rocket-chip/src/main/scala/tilelink/Broadcast.scala

// See LICENSE.SiFive for license details.

package freechips.rocketchip.tilelink

import Chisel._
import freechips.rocketchip.config.Parameters
import freechips.rocketchip.diplomacy._
import freechips.rocketchip.util._
import scala.math.{min,max}

class TLBroadcast(lineBytes: Int, numTrackers: Int = 4, bufferless: Boolean = false)(implicit p: Parameters) extends LazyModule
{
  require (lineBytes > 0 && isPow2(lineBytes))
  require (numTrackers > 0)

  val node = TLAdapterNode(
    clientFn  = { cp =>
      cp.copy(clients = Seq(TLClientParameters(
        name     = "TLBroadcast",
        sourceId = IdRange(0, 1 << log2Ceil(cp.endSourceId*4)))))
    },
    managerFn = { mp =>
      mp.copy(
        endSinkId  = numTrackers,
        managers   = mp.managers.map { m =>
          // We are the last level manager
          require (!m.supportsAcquireB)
          // We only manage addresses which are uncached
          if (m.regionType == RegionType.UNCACHED) {
            // The device had better support line transfers
            val lowerBound = max(m.supportsPutFull.min, m.supportsGet.min)
            require (!m.supportsPutFull || m.supportsPutFull.contains(lineBytes), s"${m.name} only supports PutFull(${m.supportsPutFull}), which does not include $lineBytes")
            require (!m.supportsGet     || m.supportsGet    .contains(lineBytes), s"${m.name} only supports Get(${m.supportsGet}), which does not include $lineBytes")
            m.copy(
              regionType         = RegionType.TRACKED,
              supportsAcquireB   = TransferSizes(lowerBound, lineBytes),
              supportsAcquireT   = if (m.supportsPutFull) TransferSizes(lowerBound, lineBytes) else TransferSizes.none,
              // truncate supported accesses to lineBytes (we only ever probe for one line)
              supportsPutFull    = TransferSizes(m.supportsPutFull   .min, min(m.supportsPutFull   .max, lineBytes)),
              supportsPutPartial = TransferSizes(m.supportsPutPartial.min, min(m.supportsPutPartial.max, lineBytes)),
              supportsGet        = TransferSizes(m.supportsGet       .min, min(m.supportsGet       .max, lineBytes)),
              supportsHint       = TransferSizes(m.supportsHint      .min, min(m.supportsHint      .max, lineBytes)),
              supportsArithmetic = TransferSizes(m.supportsArithmetic.min, min(m.supportsArithmetic.max, lineBytes)),
              supportsLogical    = TransferSizes(m.supportsLogical   .min, min(m.supportsLogical   .max, lineBytes)),
              fifoId             = None // trackers do not respond in FIFO order!
            )
          } else {
            m
          }
        }
      )
    }
  )

  lazy val module = new LazyModuleImp(this) {
    val io = new Bundle {
      val in  = node.bundleIn
      val out = node.bundleOut
    }

    ((io.in zip io.out) zip (node.edgesIn zip node.edgesOut)) foreach { case ((in, out), (edgeIn, edgeOut)) =>
      val clients = edgeIn.client.clients
      val managers = edgeOut.manager.managers
      val lineShift = log2Ceil(lineBytes)

      import TLBroadcastConstants._

      require (lineBytes >= edgeOut.manager.beatBytes)
      // For the probe walker, we need to identify all the caches
      val caches = clients.filter(_.supportsProbe).map(_.sourceId)
      val cache_targets = caches.map(c => UInt(c.start))

      // Create the request tracker queues
      val trackers = Seq.tabulate(numTrackers) { id =>
        Module(new TLBroadcastTracker(id, lineBytes, log2Up(caches.size+1), bufferless, edgeIn, edgeOut)).io
      }

      // We always accept E
      in.e.ready := Bool(true)
      (trackers zip UIntToOH(in.e.bits.sink).toBools) foreach { case (tracker, select) =>
        tracker.e_last := select && in.e.fire()
      }

      // Depending on the high source bits, we might transform D
      val d_high = log2Ceil(edgeIn.client.endSourceId)
      val d_what = out.d.bits.source(d_high+1, d_high)
      val d_drop = d_what === DROP
      val d_hasData = edgeOut.hasData(out.d.bits)
      val d_normal = Wire(in.d)
      val d_trackerOH = Vec(trackers.map { t => !t.idle && t.source === d_normal.bits.source }).asUInt

      assert (!out.d.valid || !d_drop || out.d.bits.opcode === TLMessages.AccessAck)

      out.d.ready := d_normal.ready || d_drop
      d_normal.valid := out.d.valid && !d_drop
      d_normal.bits := out.d.bits // truncates source
      when (d_what(1)) { // TRANSFORM_*
        d_normal.bits.opcode := Mux(d_hasData, TLMessages.GrantData, TLMessages.ReleaseAck)
        d_normal.bits.param  := Mux(d_hasData, Mux(d_what(0), TLPermissions.toT, TLPermissions.toB), UInt(0))
      }
      d_normal.bits.sink := OHToUInt(d_trackerOH)
      assert (!d_normal.valid || (d_trackerOH.orR() || d_normal.bits.opcode === TLMessages.ReleaseAck))

      // A tracker response is anything neither dropped nor a ReleaseAck
      val d_response = d_hasData || !d_what(1)
      val d_last = edgeIn.last(d_normal)
      (trackers zip d_trackerOH.toBools) foreach { case (tracker, select) =>
        tracker.d_last := select && d_normal.fire() && d_response && d_last
        tracker.probedack := select && out.d.fire() && d_drop
      }

      // Incoming C can be:
      // ProbeAck     => decrement tracker, drop 
      // ProbeAckData => decrement tracker, send out A as PutFull(DROP)
      // ReleaseData  =>                    send out A as PutFull(TRANSFORM)
      // Release      => send out D as ReleaseAck

      val c_probeack     = in.c.bits.opcode === TLMessages.ProbeAck
      val c_probeackdata = in.c.bits.opcode === TLMessages.ProbeAckData
      val c_releasedata  = in.c.bits.opcode === TLMessages.ReleaseData
      val c_release      = in.c.bits.opcode === TLMessages.Release
      val c_trackerOH    = trackers.map { t => t.line === (in.c.bits.address >> lineShift) }
      val c_trackerSrc   = Mux1H(c_trackerOH, trackers.map { _.source })

      // Decrement the tracker's outstanding probe counter
      (trackers zip c_trackerOH) foreach { case (tracker, select) =>
        tracker.probenack := in.c.fire() && c_probeack && select
      }

      val releaseack = Wire(in.d)
      val putfull = Wire(out.a)

      in.c.ready := c_probeack || Mux(c_release, releaseack.ready, putfull.ready)

      releaseack.valid := in.c.valid && c_release
      releaseack.bits  := edgeIn.ReleaseAck(in.c.bits)

      val put_what = Mux(c_releasedata, TRANSFORM_B, DROP)
      val put_who  = Mux(c_releasedata, in.c.bits.source, c_trackerSrc)
      putfull.valid := in.c.valid && (c_probeackdata || c_releasedata)
      putfull.bits := edgeOut.Put(Cat(put_what, put_who), in.c.bits.address, in.c.bits.size, in.c.bits.data)._2

      // Combine ReleaseAck or the modified D
      TLArbiter.lowest(edgeOut, in.d, releaseack, d_normal)
      // Combine the PutFull with the trackers
      TLArbiter.lowestFromSeq(edgeOut, out.a, putfull +: trackers.map(_.out_a))

      // The Probe FSM walks all caches and probes them
      val probe_todo = RegInit(UInt(0, width = max(1, caches.size)))
      val probe_line = Reg(UInt())
      val probe_perms = Reg(UInt(width = 2))
      val probe_next = probe_todo & ~(leftOR(probe_todo) << 1)
      val probe_busy = probe_todo.orR()
      val probe_target = if (caches.size == 0) UInt(0) else Mux1H(probe_next, cache_targets)

      // Probe whatever the FSM wants to do next
      in.b.valid := probe_busy
      if (caches.size != 0) {
        in.b.bits := edgeIn.Probe(probe_line << lineShift, probe_target, UInt(lineShift), probe_perms)._2
      }
      when (in.b.fire()) { probe_todo := probe_todo & ~probe_next }

      // Which cache does a request come from?
      val a_cache = if (caches.size == 0) UInt(1) else Vec(caches.map(_.contains(in.a.bits.source))).asUInt
      val a_first = edgeIn.first(in.a)

      // To accept a request from A, the probe FSM must be idle and there must be a matching tracker
      val freeTrackers = Vec(trackers.map { t => t.idle }).asUInt
      val freeTracker = freeTrackers.orR()
      val matchTrackers = Vec(trackers.map { t => t.line === in.a.bits.address >> lineShift }).asUInt
      val matchTracker = matchTrackers.orR()
      val allocTracker = freeTrackers & ~(leftOR(freeTrackers) << 1)
      val selectTracker = Mux(matchTracker, matchTrackers, allocTracker)

      val trackerReady = Vec(trackers.map(_.in_a.ready)).asUInt
      in.a.ready := (!a_first || !probe_busy) && (selectTracker & trackerReady).orR()
      (trackers zip selectTracker.toBools) foreach { case (t, select) =>
        t.in_a.valid := in.a.valid && select && (!a_first || !probe_busy)
        t.in_a.bits := in.a.bits
        t.in_a_first := a_first
        t.probe := (if (caches.size == 0) UInt(0) else Mux(a_cache.orR(), UInt(caches.size-1), UInt(caches.size)))
      }

      when (in.a.fire() && a_first) {
        probe_todo  := ~a_cache // probe all but the cache who poked us
        probe_line  := in.a.bits.address >> lineShift
        probe_perms := MuxLookup(in.a.bits.opcode, Wire(UInt(width = 2)), Array(
          TLMessages.PutFullData    -> TLPermissions.toN,
          TLMessages.PutPartialData -> TLPermissions.toN,
          TLMessages.ArithmeticData -> TLPermissions.toN,
          TLMessages.LogicalData    -> TLPermissions.toN,
          TLMessages.Get            -> TLPermissions.toB,
          TLMessages.Hint           -> MuxLookup(in.a.bits.param, Wire(UInt(width = 2)), Array(
            TLHints.PREFETCH_READ   -> TLPermissions.toB,
            TLHints.PREFETCH_WRITE  -> TLPermissions.toN)),
          TLMessages.Acquire        -> MuxLookup(in.a.bits.param, Wire(UInt(width = 2)), Array(
            TLPermissions.NtoB      -> TLPermissions.toB,
            TLPermissions.NtoT      -> TLPermissions.toN,
            TLPermissions.BtoT      -> TLPermissions.toN))))
      }

      // The outer TL connections may not be cached
      out.b.ready := Bool(true)
      out.c.valid := Bool(false)
      out.e.valid := Bool(false)
    }
  }
}

class TLBroadcastTracker(id: Int, lineBytes: Int, probeCountBits: Int, bufferless: Boolean, edgeIn: TLEdgeIn, edgeOut: TLEdgeOut) extends Module
{
  val io = new Bundle {
    val in_a_first = Bool(INPUT)
    val in_a  = Decoupled(new TLBundleA(edgeIn.bundle)).flip
    val out_a = Decoupled(new TLBundleA(edgeOut.bundle))
    val probe = UInt(INPUT, width = probeCountBits)
    val probenack = Bool(INPUT)
    val probedack = Bool(INPUT)
    val d_last = Bool(INPUT)
    val e_last = Bool(INPUT)
    val source = UInt(OUTPUT) // the source awaiting D response
    val line = UInt(OUTPUT)   // the line waiting for probes
    val idle = Bool(OUTPUT)
  }

  val lineShift = log2Ceil(lineBytes)
  import TLBroadcastConstants._

  // Only one operation can be inflight per line, because we need to be sure
  // we send the request after all the probes we sent and before all the next probes
  val got_e   = RegInit(Bool(true))
  val sent_d  = RegInit(Bool(true))
  val opcode  = Reg(io.in_a.bits.opcode)
  val param   = Reg(io.in_a.bits.param)
  val size    = Reg(io.in_a.bits.size)
  val source  = Reg(io.in_a.bits.source)
  val address = RegInit(UInt(id << lineShift, width = io.in_a.bits.address.getWidth))
  val count   = Reg(UInt(width = probeCountBits))
  val idle    = got_e && sent_d

  when (io.in_a.fire() && io.in_a_first) {
    assert (idle)
    sent_d  := Bool(false)
    got_e   := io.in_a.bits.opcode =/= TLMessages.Acquire
    opcode  := io.in_a.bits.opcode
    param   := io.in_a.bits.param
    size    := io.in_a.bits.size
    source  := io.in_a.bits.source
    address := io.in_a.bits.address
    count   := io.probe
  }
  when (io.d_last) {
    assert (!sent_d)
    sent_d := Bool(true)
  }
  when (io.e_last) {
    assert (!got_e)
    got_e := Bool(true)
  }

  when (io.probenack || io.probedack) {
    assert (count > UInt(0))
    count := count - Mux(io.probenack && io.probedack, UInt(2), UInt(1))
  }

  io.idle := idle
  io.source := source
  io.line := address >> lineShift

  val i_data = Wire(Decoupled(new TLBroadcastData(edgeIn.bundle)))
  val o_data = Queue(i_data, if (bufferless) 1 else (lineBytes / edgeIn.manager.beatBytes), pipe=bufferless)

  io.in_a.ready := (idle || !io.in_a_first) && i_data.ready
  i_data.valid := (idle || !io.in_a_first) && io.in_a.valid
  i_data.bits.mask := io.in_a.bits.mask
  i_data.bits.data := io.in_a.bits.data

  val probe_done = count === UInt(0)
  val acquire = opcode === TLMessages.Acquire

  val transform = MuxLookup(param, Wire(UInt(width = 2)), Array(
    TLPermissions.NtoB -> TRANSFORM_B,
    TLPermissions.NtoT -> TRANSFORM_T,
    TLPermissions.BtoT -> TRANSFORM_T))

  o_data.ready := io.out_a.ready && probe_done
  io.out_a.valid := o_data.valid && probe_done
  io.out_a.bits.opcode  := Mux(acquire, TLMessages.Get, opcode)
  io.out_a.bits.param   := Mux(acquire, UInt(0), param)
  io.out_a.bits.size    := size
  io.out_a.bits.source  := Cat(Mux(acquire, transform, PASS), source)
  io.out_a.bits.address := address
  io.out_a.bits.mask    := o_data.bits.mask
  io.out_a.bits.data    := o_data.bits.data
}

object TLBroadcastConstants
{
  def TRANSFORM_T = UInt(3)
  def TRANSFORM_B = UInt(2)
  def DROP        = UInt(1)
  def PASS        = UInt(0)
}

class TLBroadcastData(params: TLBundleParameters) extends TLBundleBase(params)
{
  val mask = UInt(width = params.dataBits/8)
  val data = UInt(width = params.dataBits)
}