Merge pull request #898 from freechipsproject/uncached-acquire

tilelink: it is now legal to support Acquire for UNCACHED regions

src/main/scala/amba/axi4/Test.scala

@@ -98,10 +98,10 @@ class AXI4FuzzSlave()(implicit p: Parameters) extends LazyModule with HasFuzzTar
   val node = AXI4InputNode()
   val xbar = LazyModule(new TLXbar)
   val ram  = LazyModule(new TLRAM(fuzzAddr))
-  val error= LazyModule(new TLError(ErrorParams(Seq(AddressSet(0x1800, 0xff)))))
+  val error= LazyModule(new TLError(ErrorParams(Seq(AddressSet(0x1800, 0xff)), maxTransfer = 256)))
 
   ram.node   := TLFragmenter(4, 16)(xbar.node)
-  error.node := TLFragmenter(4, 16)(xbar.node)
+  error.node := xbar.node
 
   xbar.node :=
     TLFIFOFixer()(

src/main/scala/coreplex/SystemBus.scala

@@ -36,6 +36,8 @@ class SystemBus(params: SystemBusParams)(implicit p: Parameters) extends TLBusWr
 
   val toMemoryBus: TLOutwardNode = outwardNode
 
+  val toSlave: TLOutwardNode = outwardNode
+
   def fromAsyncMasters(depth: Int = 8, sync: Int = 3): TLAsyncInwardNode = {
     val sink = LazyModule(new TLAsyncCrossingSink(depth, sync))
     inwardNode :=* sink.node

src/main/scala/devices/tilelink/BusBypass.scala

@@ -0,0 +1,152 @@
+// See LICENSE.SiFive for license details.
+
+package freechips.rocketchip.devices.tilelink
+
+import Chisel._
+import freechips.rocketchip.config.{Field, Parameters}
+import freechips.rocketchip.coreplex.HasPeripheryBus
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.util._
+import scala.math.min
+
+class TLBusBypass(beatBytes: Int)(implicit p: Parameters) extends LazyModule
+{
+  private val nodeIn = TLInputNode()
+  private val nodeOut = TLOutputNode()
+  val node = NodeHandle(nodeIn, nodeOut)
+
+  private val bar = LazyModule(new TLBusBypassBar)
+  private val everything = Seq(AddressSet(0, BigInt("ffffffffffffffffffffffffffffffff", 16))) // 128-bit
+  private val error = LazyModule(new TLError(ErrorParams(everything), beatBytes))
+  bar.node := nodeIn
+  error.node := bar.node
+  nodeOut := bar.node
+
+  lazy val module = new LazyModuleImp(this) {
+    val io = new Bundle {
+      val in = nodeIn.bundleIn
+      val out = nodeOut.bundleOut
+      val bypass = Bool(INPUT)
+    }
+    bar.module.io.bypass := io.bypass
+  }
+}
+
+private class TLBusBypassBar(implicit p: Parameters) extends LazyModule
+{
+  // The client only sees the second slave port
+  val node = TLNexusNode(
+    numClientPorts  = 2 to 2 ,
+    numManagerPorts = 1 to 1,
+    clientFn = { seq => seq(0) },
+    managerFn = { seq => seq(1) })
+
+  lazy val module = new LazyModuleImp(this) {
+    val io = new Bundle {
+      val in  = node.bundleIn
+      val out = node.bundleOut
+      val bypass = Bool(INPUT)
+    }
+
+    val in   = io.in(0)
+    val out0 = io.out(0)
+    val out1 = io.out(1)
+
+    val edge = node.edgesIn(0)
+    val bce = edge.manager.anySupportAcquireB && edge.client.anySupportProbe
+
+    // We need to be locked to the given bypass direction until all transactions stop
+    val flight = RegInit(UInt(0, width = log2Ceil(3*edge.client.endSourceId+1)))
+    val bypass = RegInit(io.bypass) // synchronous reset required
+
+    val (a_first, a_last, _) = edge.firstlast(in.a)
+    val (b_first, b_last, _) = edge.firstlast(in.b)
+    val (c_first, c_last, _) = edge.firstlast(in.c)
+    val (d_first, d_last, _) = edge.firstlast(in.d)
+    val (e_first, e_last, _) = edge.firstlast(in.e)
+
+    val (a_request, a_response) = (edge.isRequest(in.a.bits), edge.isResponse(in.a.bits))
+    val (b_request, b_response) = (edge.isRequest(in.b.bits), edge.isResponse(in.b.bits))
+    val (c_request, c_response) = (edge.isRequest(in.c.bits), edge.isResponse(in.c.bits))
+    val (d_request, d_response) = (edge.isRequest(in.d.bits), edge.isResponse(in.d.bits))
+    val (e_request, e_response) = (edge.isRequest(in.e.bits), edge.isResponse(in.e.bits))
+
+    val a_inc = in.a.fire() && a_first && a_request
+    val b_inc = in.b.fire() && b_first && b_request
+    val c_inc = in.c.fire() && c_first && c_request
+    val d_inc = in.d.fire() && d_first && d_request
+    val e_inc = in.e.fire() && e_first && e_request
+    val inc = Cat(Seq(a_inc, d_inc) ++ (if (bce) Seq(b_inc, c_inc, e_inc) else Nil))
+
+    val a_dec = in.a.fire() && a_last && a_response
+    val b_dec = in.b.fire() && b_last && b_response
+    val c_dec = in.c.fire() && c_last && c_response
+    val d_dec = in.d.fire() && d_last && d_response
+    val e_dec = in.e.fire() && e_last && e_response
+    val dec = Cat(Seq(a_dec, d_dec) ++ (if (bce) Seq(b_dec, c_dec, e_dec) else Nil))
+
+    val next_flight = flight + PopCount(inc) - PopCount(dec)
+    flight := next_flight
+
+    when (next_flight === UInt(0)) { bypass := io.bypass }
+    val stall = bypass != io.bypass
+
+    out0.a.valid := !stall && in.a.valid &&  bypass
+    out1.a.valid := !stall && in.a.valid && !bypass
+    in.a.ready   := !stall && Mux(bypass, out0.a.ready, out1.a.ready)
+    out0.a.bits  := in.a.bits
+    out1.a.bits  := in.a.bits
+
+    out0.d.ready := in.d.ready &&  bypass
+    out1.d.ready := in.d.ready && !bypass
+    in.d.valid   := Mux(bypass, out0.d.valid, out1.d.valid)
+
+    // Argh. The Bundles are not identical, so Mux on bits does not work
+    in.d.bits.opcode := Mux(bypass, out0.d.bits.opcode, out1.d.bits.opcode)
+    in.d.bits.param  := Mux(bypass, out0.d.bits.param,  out1.d.bits.param)
+    in.d.bits.size   := Mux(bypass, out0.d.bits.size,   out1.d.bits.size)
+    in.d.bits.source := Mux(bypass, out0.d.bits.source, out1.d.bits.source)
+    in.d.bits.sink   := Mux(bypass, out0.d.bits.sink,   out1.d.bits.sink)
+    in.d.bits.data   := Mux(bypass, out0.d.bits.data,   out1.d.bits.data)
+    in.d.bits.error  := Mux(bypass, out0.d.bits.error,  out1.d.bits.error)
+
+    if (bce) {
+      out0.b.ready := in.b.ready &&  bypass
+      out1.b.ready := in.b.ready && !bypass
+      in.b.valid   := Mux(bypass, out0.b.valid, out1.b.valid)
+
+      in.b.bits.opcode := Mux(bypass, out0.b.bits.opcode, out1.b.bits.opcode)
+      in.b.bits.param  := Mux(bypass, out0.b.bits.param,  out1.b.bits.param)
+      in.b.bits.size   := Mux(bypass, out0.b.bits.size,   out1.b.bits.size)
+      in.b.bits.source := Mux(bypass, out0.b.bits.source, out1.b.bits.source)
+      in.b.bits.address:= Mux(bypass, out0.b.bits.address,out1.b.bits.address)
+      in.b.bits.mask   := Mux(bypass, out0.b.bits.mask,   out1.b.bits.mask)
+      in.b.bits.data   := Mux(bypass, out0.b.bits.data,   out1.b.bits.data)
+
+      out0.c.valid := in.c.valid &&  bypass
+      out1.c.valid := in.c.valid && !bypass
+      in.c.ready   := Mux(bypass, out0.c.ready, out1.c.ready)
+      out0.c.bits  := in.c.bits
+      out1.c.bits  := in.c.bits
+
+      out0.e.valid := in.e.valid &&  bypass
+      out1.e.valid := in.e.valid && !bypass
+      in.e.ready   := Mux(bypass, out0.e.ready, out1.e.ready)
+      out0.e.bits  := in.e.bits
+      out1.e.bits  := in.e.bits
+    } else {
+      in.b.valid := Bool(false)
+      in.c.ready := Bool(true)
+      in.e.ready := Bool(true)
+
+      out0.b.ready := Bool(true)
+      out0.c.valid := Bool(false)
+      out0.e.valid := Bool(false)
+
+      out1.b.ready := Bool(true)
+      out1.c.valid := Bool(false)
+      out1.e.valid := Bool(false)
+    }
+  }
+}

src/main/scala/devices/tilelink/Error.scala

@@ -4,13 +4,13 @@ package freechips.rocketchip.devices.tilelink
 
 import Chisel._
 import freechips.rocketchip.config.{Field, Parameters}
-import freechips.rocketchip.coreplex.HasPeripheryBus
+import freechips.rocketchip.coreplex.HasSystemBus
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.util._
 import scala.math.min
 
-case class ErrorParams(address: Seq[AddressSet])
+case class ErrorParams(address: Seq[AddressSet], maxTransfer: Int = 4096)
 case object ErrorParams extends Field[ErrorParams]
 
 /** Adds a /dev/null slave that generates TL error response messages */
@@ -20,18 +20,23 @@ class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters) e
 
   val device = new SimpleDevice("error-device", Seq("sifive,error0"))
 
+  val xfer = TransferSizes(1, params.maxTransfer)
   val node = TLManagerNode(Seq(TLManagerPortParameters(
     Seq(TLManagerParameters(
       address            = address,
       resources          = device.reg("mem"),
-      supportsGet        = TransferSizes(1, beatBytes),
+      regionType         = RegionType.UNCACHED,
-      supportsPutPartial = TransferSizes(1, beatBytes),
+      supportsAcquireT   = xfer,
-      supportsPutFull    = TransferSizes(1, beatBytes),
+      supportsAcquireB   = xfer,
-      supportsArithmetic = TransferSizes(1, beatBytes),
+      supportsGet        = xfer,
-      supportsLogical    = TransferSizes(1, beatBytes),
+      supportsPutPartial = xfer,
-      supportsHint       = TransferSizes(1, beatBytes),
+      supportsPutFull    = xfer,
+      supportsArithmetic = xfer,
+      supportsLogical    = xfer,
+      supportsHint       = xfer,
       fifoId             = Some(0))), // requests are handled in order
     beatBytes  = beatBytes,
+    endSinkId  = 1, // can receive GrantAck
     minLatency = 1))) // no bypass needed for this device
 
   lazy val module = new LazyModuleImp(this) {
@@ -40,35 +45,57 @@ class TLError(params: ErrorParams, beatBytes: Int = 4)(implicit p: Parameters) e
     }
 
     import TLMessages._
-    val opcodes = Vec(AccessAck, AccessAck, AccessAckData, AccessAckData, AccessAckData, HintAck)
+    import TLPermissions._
 
+    val edge = node.edgesIn(0)
     val in = io.in(0)
     val a = Queue(in.a, 1)
-    val d = in.d
+    val c = Queue(in.c, 1)
+    val da = Wire(in.d)
+    val dc = Wire(in.d)
 
-    a.ready := d.ready
+    val a_last = edge.last(a)
-    d.valid := a.valid
+    val c_last = edge.last(c)
-    d.bits.opcode  := opcodes(a.bits.opcode)
+    val da_last = edge.last(da)
-    d.bits.param   := UInt(0)
+    val dc_last = edge.last(dc)
-    d.bits.size    := a.bits.size
-    d.bits.source  := a.bits.source
-    d.bits.sink    := UInt(0)
-    d.bits.data    := UInt(0)
-    d.bits.error   := a.bits.opcode =/= Hint // Hints may not error
 
-    // Tie off unused channels
+    a.ready := (da.ready && da_last) || !a_last
+    da.valid := a.valid && a_last
+
+    val a_opcodes = Vec(AccessAck, AccessAck, AccessAckData, AccessAckData, AccessAckData, HintAck, Grant)
+    da.bits.opcode  := a_opcodes(a.bits.opcode)
+    da.bits.param   := UInt(0)
+    da.bits.size    := a.bits.size
+    da.bits.source  := a.bits.source
+    da.bits.sink    := UInt(0)
+    da.bits.data    := UInt(0)
+    da.bits.error   := Bool(true)
+
+    c.ready := (dc.ready && dc_last) || !c_last
+    dc.valid := c.valid && c_last
+
+    dc.bits.opcode := ReleaseAck
+    dc.bits.param  := Vec(toN, toN, toB)(c.bits.param)
+    dc.bits.size   := c.bits.size
+    dc.bits.source := c.bits.source
+    dc.bits.sink   := UInt(0)
+    dc.bits.data   := UInt(0)
+    dc.bits.error  := Bool(true)
+
+    // Combine response channels
+    TLArbiter.lowest(edge, in.d, dc, da)
+
+    // We never probe or issue B requests; we are UNCACHED
     in.b.valid := Bool(false)
-    in.c.ready := Bool(true)
+
+    // Sink GrantAcks
     in.e.ready := Bool(true)
   }
 }
 
-trait HasPeripheryErrorSlave extends HasPeripheryBus {
+trait HasSystemErrorSlave extends HasSystemBus {
   private val params = p(ErrorParams)
-  private val maxXfer = min(params.address.map(_.alignment).max.toInt, 4096)
+  val error = LazyModule(new TLError(params, sbus.beatBytes))
-  val error = LazyModule(new TLError(params, pbus.beatBytes))
 
-  // Most slaves do not support a 4kB burst so this slave ends up with many more source bits than others;
+  error.node := sbus.toSlave
-  // we exclude the onerously large TLMonitor that results.
-  error.node connectButDontMonitor pbus.toLargeBurstSlave(maxXfer)
 }

src/main/scala/diplomacy/Parameters.scala

@@ -227,6 +227,7 @@ object AddressRange
 
 object AddressSet
 {
+  val everything = AddressSet(0, -1)
   def misaligned(base: BigInt, size: BigInt, tail: Seq[AddressSet] = Seq()): Seq[AddressSet] = {
     if (size == 0) tail.reverse else {
       val maxBaseAlignment = base & (-base) // 0 for infinite (LSB)

src/main/scala/system/ExampleRocketSystem.scala

@@ -14,7 +14,7 @@ class ExampleRocketSystem(implicit p: Parameters) extends RocketCoreplex
     with HasMasterAXI4MMIOPort
     with HasSlaveAXI4Port
     with HasPeripheryBootROM
-    with HasPeripheryErrorSlave {
+    with HasSystemErrorSlave {
   override lazy val module = new ExampleRocketSystemModule(this)
 }
 

src/main/scala/tilelink/CacheCork.scala

@@ -20,9 +20,8 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
       mp.copy(
         endSinkId = 1,
         managers = mp.managers.map { m => m.copy(
-          regionType         = if (m.regionType == RegionType.UNCACHED) RegionType.TRACKED else m.regionType,
+          supportsAcquireB = if (m.regionType == RegionType.UNCACHED) m.supportsGet     else m.supportsAcquireB,
-          supportsAcquireB   = m.supportsGet,
+          supportsAcquireT = if (m.regionType == RegionType.UNCACHED) m.supportsPutFull else m.supportsAcquireT)})})
-          supportsAcquireT   = m.supportsPutFull)})})
 
   lazy val module = new LazyModuleImp(this) {
     val io = new Bundle {
@@ -37,6 +36,7 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
       require (caches.size <= 1 || unsafe, "Only one caching client allowed")
       edgeOut.manager.managers.foreach { case m =>
         require (!m.supportsAcquireB || unsafe, "Cannot support caches beyond the Cork")
+        require (m.regionType <= RegionType.UNCACHED)
       }
 
       // The Cork turns [Acquire=>Get] => [AccessAckData=>GrantData]
@@ -48,10 +48,15 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
       //   Put: 1, Release: 0 => AccessAck
       //   *: 0, Acquire: 1 => AccessAckData
 
-      // Take requests from A to A
+      // Take requests from A to A or D (if BtoT Acquire)
-      val isPut = in.a.bits.opcode === PutFullData || in.a.bits.opcode === PutPartialData
       val a_a = Wire(out.a)
-      a_a <> in.a
+      val a_d = Wire(in.d)
+      val isPut = in.a.bits.opcode === PutFullData || in.a.bits.opcode === PutPartialData
+      val toD = in.a.bits.opcode === Acquire && in.a.bits.param === TLPermissions.BtoT
+      in.a.ready := Mux(toD, a_d.ready, a_a.ready)
+
+      a_a.valid := in.a.valid && !toD
+      a_a.bits := in.a.bits
       a_a.bits.source := in.a.bits.source << 1 | Mux(isPut, UInt(1), UInt(0))
 
       // Transform Acquire into Get
@@ -61,26 +66,27 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
         a_a.bits.source := in.a.bits.source << 1 | UInt(1)
       }
 
+      // Upgrades are instantly successful
+      a_d.valid := in.a.valid && toD
+      a_d.bits := edgeIn.Grant(
+        fromSink = UInt(0),
+        toSource = in.a.bits.source,
+        lgSize   = in.a.bits.size,
+        capPermissions = TLPermissions.toT)
+
       // Take ReleaseData from C to A; Release from C to D
       val c_a = Wire(out.a)
       c_a.valid := in.c.valid && in.c.bits.opcode === ReleaseData
-      c_a.bits.opcode  := PutFullData
+      c_a.bits := edgeOut.Put(
-      c_a.bits.param   := UInt(0)
+        fromSource = in.c.bits.source << 1,
-      c_a.bits.size    := in.c.bits.size
+        toAddress  = in.c.bits.address,
-      c_a.bits.source  := in.c.bits.source << 1
+        lgSize     = in.c.bits.size,
-      c_a.bits.address := in.c.bits.address
+        data       = in.c.bits.data)._2
-      c_a.bits.mask    := edgeOut.mask(in.c.bits.address, in.c.bits.size)
-      c_a.bits.data    := in.c.bits.data
 
+      // Releases without Data succeed instantly
       val c_d = Wire(in.d)
       c_d.valid := in.c.valid && in.c.bits.opcode === Release
-      c_d.bits.opcode  := ReleaseAck
+      c_d.bits := edgeIn.ReleaseAck(in.c.bits)
-      c_d.bits.param   := UInt(0)
-      c_d.bits.size    := in.c.bits.size
-      c_d.bits.source  := in.c.bits.source
-      c_d.bits.sink    := UInt(0)
-      c_d.bits.data    := UInt(0)
-      c_d.bits.error   := Bool(false)
 
       assert (!in.c.valid || in.c.bits.opcode === Release || in.c.bits.opcode === ReleaseData)
       in.c.ready := Mux(in.c.bits.opcode === Release, c_d.ready, c_a.ready)
@@ -99,7 +105,10 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
 
       when (out.d.bits.opcode === AccessAckData && out.d.bits.source(0)) {
         d_d.bits.opcode := GrantData
-        d_d.bits.param  := TLPermissions.toT
+        // On Grant error, you do NOT get the permissions you asked for.
+        // We only enter this case from NtoT or NtoB, so that means use toN.
+        // (the BtoT case was handled by a_d)
+        d_d.bits.param  := Mux(out.d.bits.error, TLPermissions.toN, TLPermissions.toT)
       }
       when (out.d.bits.opcode === AccessAck && !out.d.bits.source(0)) {
         d_d.bits.opcode := ReleaseAck
@@ -107,7 +116,7 @@ class TLCacheCork(unsafe: Boolean = false)(implicit p: Parameters) extends LazyM
 
       // Combine the sources of messages into the channels
       TLArbiter(TLArbiter.lowestIndexFirst)(out.a, (edgeOut.numBeats1(c_a.bits), c_a), (edgeOut.numBeats1(a_a.bits), a_a))
-      TLArbiter(TLArbiter.lowestIndexFirst)(in.d,  (edgeIn .numBeats1(d_d.bits), d_d), (UInt(0), Queue(c_d, 2)))
+      TLArbiter(TLArbiter.lowestIndexFirst)(in.d,  (edgeIn .numBeats1(d_d.bits), d_d), (UInt(0), Queue(c_d, 2)), (UInt(0), Queue(a_d, 2)))
 
       // Tie off unused ports
       in.b.valid := Bool(false)

src/main/scala/tilelink/Parameters.scala

@@ -38,8 +38,9 @@ case class TLManagerParameters(
   require (supportsAcquireB.contains(supportsAcquireT))
 
   // Make sure that the regionType agrees with the capabilities
-  require ((regionType == RegionType.CACHED || regionType == RegionType.TRACKED) != supportsAcquireB.none)
+  require (!supportsAcquireB || regionType >= RegionType.UNCACHED) // acquire -> uncached, tracked, cached
-  require (regionType != RegionType.UNCACHED || supportsGet)
+  require (regionType <= RegionType.UNCACHED || supportsAcquireB)  // tracked, cached -> acquire
+  require (regionType != RegionType.UNCACHED || supportsGet) // uncached -> supportsGet
 
   val name = nodePath.lastOption.map(_.lazyModule.name).getOrElse("disconnected")
   val maxTransfer = List( // Largest supported transfer of all types