tilelink2: broadcast coherence manager

src/main/scala/uncore/tilelink2/Broadcast.scala

@@ -0,0 +1,297 @@
+// See LICENSE for license details.
+
+package uncore.tilelink2
+
+import Chisel._
+import diplomacy._
+import scala.math.{min,max}
+
+class TLBroadcast(lineBytes: Int, numTrackers: Int = 4) extends LazyModule
+{
+  require (lineBytes > 0 && isPow2(lineBytes))
+  require (numTrackers > 0)
+
+  val node = TLAdapterNode(
+    clientFn  = { case Seq(cp) =>
+      cp.copy(clients = Seq(TLClientParameters(
+        sourceId = IdRange(0, 1 << log2Ceil(cp.endSourceId*4)))))
+    },
+    managerFn = { case Seq(mp) =>
+      mp.copy(
+        endSinkId  = numTrackers,
+        minLatency = 1,
+        managers   = mp.managers.map { m =>
+          // We are the last level manager
+          require (m.regionType != RegionType.CACHED)
+          require (m.regionType != RegionType.TRACKED)
+          require (!m.supportsAcquire)
+          // We only manage addresses which are uncached
+          if (m.regionType == RegionType.UNCACHED) {
+            // The device had better support line transfers
+            require (!m.supportsPutFull    || m.supportsPutFull   .contains(lineBytes))
+            require (!m.supportsPutPartial || m.supportsPutPartial.contains(lineBytes))
+            require (!m.supportsGet        || m.supportsGet       .contains(lineBytes))
+            m.copy(
+              regionType         = RegionType.TRACKED,
+              supportsAcquire    = TransferSizes(1, lineBytes),
+              // truncate supported accesses to lineBytes (we only ever probe for one line)
+              supportsPutFull    = TransferSizes(m.supportsPutFull   .min, min(m.supportsPutFull   .max, lineBytes)),
+              supportsPutPartial = TransferSizes(m.supportsPutPartial.min, min(m.supportsPutPartial.max, lineBytes)),
+              supportsGet        = TransferSizes(m.supportsGet       .min, min(m.supportsGet       .max, lineBytes)),
+              fifoId             = None // trackers do not respond in FIFO order!
+            )
+          } else {
+            m
+          }
+        }
+      )
+    }
+  )
+
+  lazy val module = new LazyModuleImp(this) {
+    val io = new Bundle {
+      val in  = node.bundleIn
+      val out = node.bundleOut
+    }
+
+    val in = io.in(0)
+    val out = io.out(0)
+    val edgeIn = node.edgesIn(0)
+    val edgeOut = node.edgesOut(0)
+    val clients = edgeIn.client.clients
+    val managers = edgeOut.manager.managers
+    val lineShift = log2Ceil(lineBytes)
+
+    import TLBroadcastConstants._
+
+    require (lineBytes >= edgeOut.manager.beatBytes)
+    // For the probe walker, we need to identify all the caches
+    val caches = clients.filter(_.supportsProbe).map(_.sourceId)
+    val cache_targets = Vec(caches.map(c => UInt(c.start)))
+
+    // Create the request tracker queues
+    val trackers = Seq.tabulate(numTrackers) { id =>
+      Module(new TLBroadcastTracker(id, lineBytes, log2Up(caches.size), edgeIn, edgeOut)).io
+    }
+
+    // We always accept E
+    in.e.ready := Bool(true)
+    (trackers zip UIntToOH(in.e.bits.sink).toBools) foreach { case (tracker, select) =>
+      tracker.e_last := select && in.e.fire()
+    }
+
+    // Depending on the high source bits, we might transform D
+    val d_high = 1 << log2Ceil(edgeIn.client.endSourceId)
+    val d_what = out.d.bits.source(d_high+1, d_high)
+    val d_drop = d_what === DROP
+    val d_hasData = edgeOut.hasData(out.d.bits)
+    val d_normal = Wire(in.d)
+    val d_trackerOH = Vec(trackers.map { t => !t.idle && t.source === d_normal.bits.source }).asUInt
+
+    out.d.ready := d_normal.ready || d_drop
+    d_normal.valid := out.d.valid && !d_drop
+    d_normal.bits := out.d.bits // truncates source
+    when (d_what(1)) { // TRANSFORM_*
+      d_normal.bits.opcode := Mux(d_hasData, TLMessages.GrantData, TLMessages.ReleaseAck)
+      d_normal.bits.param  := Mux(d_hasData, Mux(d_what(0), TLPermissions.toT, TLPermissions.toB), UInt(0))
+    }
+    d_normal.bits.sink := OHToUInt(d_trackerOH)
+    assert (!d_normal.valid || d_trackerOH.orR())
+
+    // A tracker response is anything neither dropped nor a ReleaseAck
+    val d_response = d_hasData || !d_what(1)
+    val (_, d_last, _) = edgeIn.firstlast(d_normal)
+    (trackers zip d_trackerOH.toBools) foreach { case (tracker, select) =>
+      tracker.d_last := select && d_normal.fire() && d_response && d_last
+    }
+
+    // Incoming C can be:
+    // ProbeAck     => decrement tracker, drop 
+    // ProbeAckData => decrement tracker, send out A as PutFull(DROP)
+    // ReleaseData  =>                    send out A as PutFull(TRANSFORM)
+    // Release      => send out D as ReleaseAck
+
+    val c_probeack     = in.c.bits.opcode === TLMessages.ProbeAck
+    val c_probeackdata = in.c.bits.opcode === TLMessages.ProbeAckData
+    val c_releasedata  = in.c.bits.opcode === TLMessages.ReleaseData
+    val c_release      = in.c.bits.opcode === TLMessages.Release
+
+    // Decrement the tracker's outstanding probe counter
+    val c_decrement = in.c.fire() && (c_probeack || c_probeackdata)
+    val (_, c_last, _) = edgeIn.firstlast(in.c)
+    trackers foreach { tracker =>
+      tracker.probeack := c_decrement && c_last && tracker.line === (in.c.bits.address >> lineShift)
+    }
+
+    val releaseack = Wire(in.d)
+    val putfull = Wire(out.a)
+
+    in.c.ready := c_probeack || Mux(c_release, releaseack.ready, putfull.ready)
+
+    releaseack.valid := in.c.valid && c_release
+    releaseack.bits  := edgeIn.ReleaseAck(in.c.bits.address, UInt(0), in.c.bits.source, in.c.bits.size)
+
+    val put_what = Mux(c_releasedata, TRANSFORM_B, DROP)
+    putfull.valid := in.c.valid && (c_probeackdata || c_releasedata)
+    putfull.bits := edgeOut.Put(Cat(put_what, in.c.bits.source), in.c.bits.address, in.c.bits.size, in.c.bits.data)._2
+
+    // Combine ReleaseAck or the modified D
+    TLArbiter(TLArbiter.lowestIndexFirst)(in.d, (UInt(0), releaseack), (edgeOut.numBeats1(d_normal.bits), d_normal))
+    // Combine the PutFull with the trackers
+    TLArbiter(TLArbiter.lowestIndexFirst)(out.a, 
+      ((edgeOut.numBeats1(putfull.bits), putfull) +:
+       trackers.map { t => (edgeOut.numBeats1(t.out_a.bits), t.out_a) }):_*)
+
+    // The Probe FSM walks all caches and probes them
+    val probe_todo = RegInit(UInt(0, width = caches.size))
+    val probe_line = Reg(UInt())
+    val probe_perms = Reg(UInt(width = 2))
+    val probe_next = probe_todo & ~(leftOR(probe_todo) << 1)
+    val probe_busy = probe_todo.orR()
+    val probe_target = Mux1H(probe_next, cache_targets)
+
+    // Probe whatever the FSM wants to do next
+    in.b.valid := probe_busy
+    in.b.bits := edgeIn.Probe(probe_line << lineShift, probe_target, UInt(lineShift), probe_perms)._2
+    when (in.b.fire()) { probe_todo := probe_todo & ~probe_next }
+
+    // Which cache does a request come from?
+    val a_cache = Vec(caches.map(_.contains(in.a.bits.source))).asUInt
+    val (a_first, _, _) = edgeIn.firstlast(in.a)
+
+    // To accept a request from A, the probe FSM must be idle and there must be a matching tracker
+    val freeTrackers = Vec(trackers.map { t => t.idle }).asUInt
+    val freeTracker = freeTrackers.orR()
+    val matchTrackers = Vec(trackers.map { t => t.line === in.a.bits.address >> lineShift }).asUInt
+    val matchTracker = matchTrackers.orR()
+    val allocTracker = freeTrackers & ~(leftOR(freeTrackers) << 1)
+    val selectTracker = Mux(matchTracker, matchTrackers, allocTracker)
+
+    val trackerReady = Vec(trackers.map(_.in_a.ready)).asUInt
+    in.a.ready := (!a_first || !probe_busy) && (selectTracker & trackerReady).orR()
+    (trackers zip selectTracker.toBools) foreach { case (t, select) =>
+      t.in_a.valid := in.a.valid && select && (!a_first || !probe_busy)
+      t.in_a.bits := in.a.bits
+      t.in_a_first := a_first
+      t.probe := Mux(a_cache.orR(), UInt(caches.size-1), UInt(caches.size))
+    }
+
+    when (in.a.fire() && a_first) {
+      probe_todo  := ~a_cache // probe all but the cache who poked us
+      probe_line  := in.a.bits.address >> lineShift
+      probe_perms := MuxLookup(in.a.bits.opcode, Wire(UInt(width = 2)), Array(
+        TLMessages.PutFullData    -> TLPermissions.toN,
+        TLMessages.PutPartialData -> TLPermissions.toN,
+        TLMessages.ArithmeticData -> TLPermissions.toN,
+        TLMessages.LogicalData    -> TLPermissions.toN,
+        TLMessages.Get            -> TLPermissions.toB,
+        TLMessages.Hint           -> MuxLookup(in.a.bits.param, Wire(UInt(width = 2)), Array(
+          TLHints.PREFETCH_READ   -> TLPermissions.toB,
+          TLHints.PREFETCH_WRITE  -> TLPermissions.toN)),
+        TLMessages.Acquire        -> MuxLookup(in.a.bits.param, Wire(UInt(width = 2)), Array(
+          TLPermissions.NtoB      -> TLPermissions.toB,
+          TLPermissions.NtoT      -> TLPermissions.toN,
+          TLPermissions.BtoT      -> TLPermissions.toN))))
+    }
+
+    // The outer TL connections may not be cached
+    out.b.ready := Bool(true)
+    out.c.valid := Bool(false)
+    out.e.valid := Bool(false)
+  }
+}
+
+class TLBroadcastTracker(id: Int, lineBytes: Int, probeCountBits: Int, edgeIn: TLEdgeIn, edgeOut: TLEdgeOut) extends Module
+{
+  val io = new Bundle {
+    val in_a_first = Bool(INPUT)
+    val in_a  = Decoupled(new TLBundleA(edgeIn.bundle)).flip
+    val out_a = Decoupled(new TLBundleA(edgeOut.bundle))
+    val probe = UInt(INPUT, width = probeCountBits)
+    val probeack = Bool(INPUT)
+    val d_last = Bool(INPUT)
+    val e_last = Bool(INPUT)
+    val source = UInt(OUTPUT) // the source awaiting D response
+    val line = UInt(OUTPUT)   // the line waiting for probes
+    val idle = Bool(OUTPUT)
+  }
+
+  val lineShift = log2Ceil(lineBytes)
+  import TLBroadcastConstants._
+
+  // Only one operation can be inflight per line, because we need to be sure
+  // we send the request after all the probes we sent and before all the next probes
+  val idle    = RegInit(Bool(true))
+  val opcode  = Reg(io.in_a.bits.opcode)
+  val param   = Reg(io.in_a.bits.param)
+  val size    = Reg(io.in_a.bits.size)
+  val source  = Reg(io.in_a.bits.source)
+  val address = RegInit(UInt(id << lineShift, width = io.in_a.bits.address.getWidth))
+  val count   = Reg(UInt(width = probeCountBits))
+
+  when (io.in_a.fire() && io.in_a_first) {
+    assert (idle)
+    idle    := Bool(false)
+    opcode  := io.in_a.bits.opcode
+    param   := io.in_a.bits.param
+    size    := io.in_a.bits.size
+    source  := io.in_a.bits.source
+    address := io.in_a.bits.address
+    count   := io.probe
+  }
+  when (io.d_last) {
+    assert (!idle)
+    idle := opcode =/= TLMessages.Acquire
+  }
+  when (io.e_last) {
+    assert (!idle)
+    idle := Bool(true)
+  }
+  when (io.probeack) {
+    assert (count > UInt(0))
+    count := count - UInt(1)
+  }
+
+  io.idle := idle
+  io.source := source
+  io.line := address >> lineShift
+
+  class Data extends Bundle {
+    val mask = io.in_a.bits.mask.cloneType
+    val data = io.in_a.bits.data.cloneType
+  }
+
+  val i_data = Wire(Decoupled(new Data))
+  val o_data = Queue(i_data, lineBytes / edgeIn.manager.beatBytes)
+
+  io.in_a.ready := (idle || !io.in_a_first) && i_data.ready
+  i_data.valid := (idle || !io.in_a_first) && io.in_a.valid
+  i_data.bits.mask := io.in_a.bits.mask
+  i_data.bits.data := io.in_a.bits.data
+
+  val probe_done = count === UInt(0)
+  val acquire = opcode === TLMessages.Acquire
+
+  val transform = MuxLookup(param, Wire(UInt(width = 2)), Array(
+    TLPermissions.NtoB -> TRANSFORM_B,
+    TLPermissions.NtoT -> TRANSFORM_T,
+    TLPermissions.BtoT -> TRANSFORM_T))
+
+  o_data.ready := io.out_a.ready && probe_done
+  io.out_a.valid := o_data.valid && probe_done
+  io.out_a.bits.opcode  := Mux(acquire, TLMessages.Get, opcode)
+  io.out_a.bits.param   := Mux(acquire, UInt(0), param)
+  io.out_a.bits.size    := size
+  io.out_a.bits.source  := Cat(Mux(acquire, transform, PASS), source)
+  io.out_a.bits.address := address
+  io.out_a.bits.mask    := o_data.bits.mask
+  io.out_a.bits.data    := o_data.bits.data
+}
+
+object TLBroadcastConstants
+{
+  val TRANSFORM_T = UInt(3)
+  val TRANSFORM_B = UInt(2)
+  val DROP        = UInt(1)
+  val PASS        = UInt(0)
+}

src/main/scala/uncore/tilelink2/Bundles.scala

@@ -88,6 +88,12 @@ object TLAtomics
   def isLogical(x: UInt) = x <= SWAP
 }
 
+object TLHints
+{
+  val PREFETCH_READ  = UInt(0)
+  val PREFETCH_WRITE = UInt(1)
+}
+
 sealed trait TLChannel extends TLBundleBase {
   val channelName: String
 }