Support for uncached sub-block reads and writes, major TileLink and CoherencePolicy refactor.
This commit is contained in:
Henry Cook
@ -2,32 +2,26 @@
|
||||
|
||||
package uncore
|
||||
import Chisel._
|
||||
import scala.reflect.ClassTag
|
||||
|
||||
case object CacheName extends Field[String]
|
||||
case object NSets extends Field[Int]
|
||||
case object NWays extends Field[Int]
|
||||
case object BlockOffBits extends Field[Int]
|
||||
case object RowBits extends Field[Int]
|
||||
case object WordBits extends Field[Int]
|
||||
case object Replacer extends Field[() => ReplacementPolicy]
|
||||
case object AmoAluOperandBits extends Field[Int]
|
||||
|
||||
abstract trait CacheParameters extends UsesParameters {
|
||||
val paddrBits = params(PAddrBits)
|
||||
val vaddrBits = params(VAddrBits)
|
||||
val pgIdxBits = params(PgIdxBits)
|
||||
val nSets = params(NSets)
|
||||
val blockOffBits = params(BlockOffBits)
|
||||
val idxBits = log2Up(nSets)
|
||||
val untagBits = blockOffBits + idxBits
|
||||
val tagBits = paddrBits - untagBits
|
||||
val tagBits = params(PAddrBits) - untagBits
|
||||
val nWays = params(NWays)
|
||||
val wayBits = log2Up(nWays)
|
||||
val isDM = nWays == 1
|
||||
val wordBits = params(WordBits)
|
||||
val wordBytes = wordBits/8
|
||||
val wordOffBits = log2Up(wordBytes)
|
||||
val rowBits = params(RowBits)
|
||||
val rowWords = rowBits/wordBits
|
||||
val rowBytes = rowBits/8
|
||||
val rowOffBits = log2Up(rowBytes)
|
||||
}
|
||||
@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters {
|
||||
abstract class CacheBundle extends Bundle with CacheParameters
|
||||
abstract class CacheModule extends Module with CacheParameters
|
||||
|
||||
class StoreGen(typ: Bits, addr: Bits, dat: Bits) {
|
||||
val byte = typ === MT_B || typ === MT_BU
|
||||
val half = typ === MT_H || typ === MT_HU
|
||||
val word = typ === MT_W || typ === MT_WU
|
||||
def mask =
|
||||
Mux(byte, Bits( 1) << addr(2,0),
|
||||
Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)),
|
||||
Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)),
|
||||
Bits(255))))
|
||||
def data =
|
||||
Mux(byte, Fill(8, dat( 7,0)),
|
||||
Mux(half, Fill(4, dat(15,0)),
|
||||
wordData))
|
||||
lazy val wordData =
|
||||
Mux(word, Fill(2, dat(31,0)),
|
||||
dat)
|
||||
}
|
||||
|
||||
class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) {
|
||||
val t = new StoreGen(typ, addr, dat)
|
||||
val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D
|
||||
|
||||
val wordShift = Mux(addr(2), dat(63,32), dat(31,0))
|
||||
val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift)
|
||||
val halfShift = Mux(addr(1), word(31,16), word(15,0))
|
||||
val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift)
|
||||
val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0)))
|
||||
val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift)
|
||||
}
|
||||
|
||||
class AMOALU extends CacheModule {
|
||||
val operandBits = params(AmoAluOperandBits)
|
||||
require(operandBits == 64)
|
||||
val io = new Bundle {
|
||||
val addr = Bits(INPUT, blockOffBits)
|
||||
val cmd = Bits(INPUT, M_SZ)
|
||||
val typ = Bits(INPUT, MT_SZ)
|
||||
val lhs = Bits(INPUT, operandBits)
|
||||
val rhs = Bits(INPUT, operandBits)
|
||||
val out = Bits(OUTPUT, operandBits)
|
||||
}
|
||||
|
||||
val storegen = new StoreGen(io.typ, io.addr, io.rhs)
|
||||
val rhs = storegen.wordData
|
||||
|
||||
val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX
|
||||
val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU
|
||||
val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU
|
||||
val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization:
|
||||
io.typ === MT_B || io.typ === MT_BU
|
||||
|
||||
val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31))
|
||||
val adder_out = (io.lhs & mask).toUInt + (rhs & mask)
|
||||
|
||||
val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63))
|
||||
val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63))
|
||||
val lt_lo = io.lhs(31,0) < rhs(31,0)
|
||||
val lt_hi = io.lhs(63,32) < rhs(63,32)
|
||||
val eq_hi = io.lhs(63,32) === rhs(63,32)
|
||||
val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo)
|
||||
val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs))
|
||||
|
||||
val out = Mux(io.cmd === M_XA_ADD, adder_out,
|
||||
Mux(io.cmd === M_XA_AND, io.lhs & rhs,
|
||||
Mux(io.cmd === M_XA_OR, io.lhs | rhs,
|
||||
Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs,
|
||||
Mux(Mux(less, min, max), io.lhs,
|
||||
storegen.data)))))
|
||||
|
||||
val wmask = FillInterleaved(8, storegen.mask)
|
||||
io.out := wmask & out | ~wmask & io.lhs
|
||||
}
|
||||
|
||||
abstract class ReplacementPolicy {
|
||||
def way: UInt
|
||||
def miss: Unit
|
||||
@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule {
|
||||
io.write.ready := !rst
|
||||
}
|
||||
|
||||
abstract trait L2HellaCacheParameters extends CacheParameters
|
||||
with CoherenceAgentParameters {
|
||||
abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters {
|
||||
val idxMSB = idxBits-1
|
||||
val idxLSB = 0
|
||||
val refillCyclesPerBeat = params(TLDataBits)/rowBits
|
||||
val refillCycles = refillCyclesPerBeat*params(TLDataBeats)
|
||||
require(refillCyclesPerBeat == 1)
|
||||
}
|
||||
|
||||
abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters
|
||||
abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters
|
||||
abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters
|
||||
abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters {
|
||||
def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = {
|
||||
val (cnt, cnt_done) =
|
||||
Counter(inc && data.hasMultibeatData(), tlDataBeats)
|
||||
val done = (inc && !data.hasMultibeatData()) || cnt_done
|
||||
(cnt, done)
|
||||
}
|
||||
def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = {
|
||||
connectDataBeatCounter(in.fire(), in.bits.payload)
|
||||
}
|
||||
def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = {
|
||||
connectDataBeatCounter(in.fire(), in.bits.payload)._2
|
||||
}
|
||||
def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = {
|
||||
connectDataBeatCounter(in.fire(), in.bits)
|
||||
}
|
||||
def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = {
|
||||
connectDataBeatCounter(in.valid, in.bits)._2
|
||||
}
|
||||
}
|
||||
|
||||
trait HasL2Id extends Bundle with CoherenceAgentParameters {
|
||||
val id = UInt(width = log2Up(nTransactors + 1))
|
||||
@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle {
|
||||
val way_en = Bits(width = nWays)
|
||||
}
|
||||
|
||||
trait HasL2Data extends HasTileLinkData {
|
||||
def hasData(dummy: Int = 0) = Bool(true)
|
||||
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1)
|
||||
}
|
||||
|
||||
object L2Metadata {
|
||||
def apply(tag: Bits, coh: ManagerMetadata) = {
|
||||
val meta = new L2Metadata
|
||||
@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule {
|
||||
io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en)
|
||||
}
|
||||
|
||||
class L2DataReadReq extends L2HellaCacheBundle with HasL2Id {
|
||||
class L2DataReadReq extends L2HellaCacheBundle
|
||||
with HasCacheBlockAddress
|
||||
with HasTileLinkBeatId
|
||||
with HasL2Id {
|
||||
val way_en = Bits(width = nWays)
|
||||
val addr = Bits(width = tlAddrBits)
|
||||
}
|
||||
|
||||
class L2DataWriteReq extends L2DataReadReq {
|
||||
class L2DataWriteReq extends L2DataReadReq
|
||||
with HasL2Data {
|
||||
val wmask = Bits(width = tlWriteMaskBits)
|
||||
val data = Bits(width = tlDataBits)
|
||||
}
|
||||
|
||||
class L2DataResp extends Bundle with HasL2Id with TileLinkParameters {
|
||||
val data = Bits(width = tlDataBits)
|
||||
}
|
||||
class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data
|
||||
|
||||
trait HasL2DataReadIO extends L2HellaCacheBundle {
|
||||
val read = Decoupled(new L2DataReadReq)
|
||||
@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule {
|
||||
val wmask = FillInterleaved(8, io.write.bits.wmask)
|
||||
val reg_raddr = Reg(UInt())
|
||||
val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true)
|
||||
val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr)
|
||||
val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr)
|
||||
val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat)
|
||||
val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat)
|
||||
|
||||
when (io.write.bits.way_en.orR && io.write.valid) {
|
||||
array.write(waddr, io.write.bits.data, wmask)
|
||||
@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule {
|
||||
|
||||
io.resp.valid := ShiftRegister(io.read.fire(), 1)
|
||||
io.resp.bits.id := ShiftRegister(io.read.bits.id, 1)
|
||||
io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1)
|
||||
io.resp.bits.data := array(reg_raddr)
|
||||
io.read.ready := !io.write.valid
|
||||
io.write.ready := Bool(true)
|
||||
@ -261,10 +353,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
|
||||
}
|
||||
|
||||
// Wiring helper funcs
|
||||
def doOutputArbitration[T <: Data](out: DecoupledIO[T],
|
||||
ins: Seq[DecoupledIO[T]],
|
||||
count: Int = 1,
|
||||
lock: T => Bool = (a: T) => Bool(true)) {
|
||||
def doOutputArbitration[T <: Data](
|
||||
out: DecoupledIO[T],
|
||||
ins: Seq[DecoupledIO[T]],
|
||||
count: Int = 1,
|
||||
lock: T => Bool = (a: T) => Bool(true)) {
|
||||
val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock))
|
||||
out <> arb.io.out
|
||||
arb.io.in zip ins map { case (a, in) => a <> in }
|
||||
@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
|
||||
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe)
|
||||
|
||||
// Wire grant reply to initiating client
|
||||
def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload)
|
||||
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _)
|
||||
doOutputArbitration(
|
||||
io.inner.grant,
|
||||
trackerList.map(_.io.inner.grant),
|
||||
tlDataBeats,
|
||||
(m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData())
|
||||
|
||||
// Create an arbiter for the one memory port
|
||||
val outerList = trackerList.map(_.io.outer) :+ wb.io.outer
|
||||
@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
|
||||
// Wire local memories
|
||||
doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read))
|
||||
doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write))
|
||||
doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats)
|
||||
doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats)
|
||||
doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read)
|
||||
doOutputArbitration(io.data.write, trackerList.map(_.io.data.write))
|
||||
doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp))
|
||||
doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp)
|
||||
}
|
||||
|
||||
class L2WritebackReq extends L2HellaCacheBundle
|
||||
with HasL2Id {
|
||||
val addr = UInt(width = tlAddrBits)
|
||||
val addr_block = UInt(width = tlBlockAddrBits)
|
||||
val coh = new ManagerMetadata
|
||||
val way_en = Bits(width = nWays)
|
||||
}
|
||||
@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
val has_release_match = Bool(OUTPUT)
|
||||
val data = new L2DataRWIO
|
||||
}
|
||||
val c_acq = io.inner.acquire.bits
|
||||
val c_rel = io.inner.release.bits
|
||||
val c_gnt = io.inner.grant.bits
|
||||
val cacq = io.inner.acquire.bits
|
||||
val crel = io.inner.release.bits
|
||||
val cgnt = io.inner.grant.bits
|
||||
val c_ack = io.inner.finish.bits
|
||||
val m_gnt = io.outer.grant.bits
|
||||
val mgnt = io.outer.grant.bits
|
||||
|
||||
val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5)
|
||||
val state = Reg(init=s_idle)
|
||||
|
||||
val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
|
||||
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
|
||||
val xact_coh = Reg{ new ManagerMetadata }
|
||||
val xact_way_en = Reg{ Bits(width = nWays) }
|
||||
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
|
||||
@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
val pending_probes = Reg(init = co.dir.flush)
|
||||
val curr_p_id = co.dir.next(pending_probes)
|
||||
|
||||
val (crel_data_cnt, crel_data_done) =
|
||||
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
|
||||
val (outer_data_write_cnt, outer_data_write_done) =
|
||||
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
|
||||
val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats)
|
||||
val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats)
|
||||
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
|
||||
val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
|
||||
val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats)
|
||||
val resp_data_done = connectIncomingDataBeatCounter(io.data.resp)
|
||||
|
||||
io.has_release_match := !co.isVoluntary(c_rel.payload) &&
|
||||
co.isCoherenceConflict(xact_addr, c_rel.payload.addr) &&
|
||||
io.has_release_match := !crel.payload.isVoluntary() &&
|
||||
co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) &&
|
||||
(state === s_probe)
|
||||
|
||||
val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src)
|
||||
val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src)
|
||||
|
||||
io.outer.acquire.valid := Bool(false)
|
||||
io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr,
|
||||
UInt(trackerId),
|
||||
xact_data(outer_data_write_cnt)),
|
||||
io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock(
|
||||
client_xact_id = UInt(trackerId),
|
||||
addr_block = xact_addr_block,
|
||||
addr_beat = macq_data_cnt,
|
||||
data = xact_data(macq_data_cnt)),
|
||||
{ case TLId => outerId })
|
||||
io.outer.grant.ready := Bool(false) // Never gets mgnts
|
||||
|
||||
io.inner.probe.valid := Bool(false)
|
||||
io.inner.probe.bits.header.src := UInt(bankId)
|
||||
io.inner.probe.bits.header.dst := curr_p_id
|
||||
io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr)
|
||||
io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block)
|
||||
|
||||
io.inner.grant.valid := Bool(false)
|
||||
io.inner.acquire.ready := Bool(false)
|
||||
@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
io.data.read.valid := Bool(false)
|
||||
io.data.read.bits.id := UInt(trackerId)
|
||||
io.data.read.bits.way_en := xact_way_en
|
||||
io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt)
|
||||
io.data.read.bits.addr_block := xact_addr_block
|
||||
io.data.read.bits.addr_beat := read_data_cnt
|
||||
io.data.write.valid := Bool(false)
|
||||
|
||||
io.wb.req.ready := Bool(false)
|
||||
@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
is(s_idle) {
|
||||
io.wb.req.ready := Bool(true)
|
||||
when(io.wb.req.valid) {
|
||||
xact_addr := io.wb.req.bits.addr
|
||||
xact_addr_block := io.wb.req.bits.addr_block
|
||||
xact_coh := io.wb.req.bits.coh
|
||||
xact_way_en := io.wb.req.bits.way_en
|
||||
xact_id := io.wb.req.bits.id
|
||||
@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
when(io.inner.release.valid) {
|
||||
xact_coh := next_coh_on_rel
|
||||
// Handle released dirty data
|
||||
when(co.messageHasData(c_rel.payload)) {
|
||||
when(crel.payload.hasData()) {
|
||||
crel_had_data := Bool(true)
|
||||
xact_data(crel_data_cnt) := c_rel.payload.data
|
||||
xact_data(crel.payload.addr_beat) := crel.payload.data
|
||||
}
|
||||
// We don't decrement release_count until we've received all the data beats.
|
||||
when(!co.messageHasData(c_rel.payload) || crel_data_done) {
|
||||
when(!crel.payload.hasData() || crel_data_done) {
|
||||
release_count := release_count - UInt(1)
|
||||
}
|
||||
}
|
||||
@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
|
||||
}
|
||||
is(s_data_read) {
|
||||
io.data.read.valid := Bool(true)
|
||||
when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data }
|
||||
when(local_data_read_done) { state := s_data_resp }
|
||||
when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data }
|
||||
when(read_data_done) { state := s_data_resp }
|
||||
}
|
||||
is(s_data_resp) {
|
||||
when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data }
|
||||
when(local_data_resp_done) { state := s_outer_write }
|
||||
when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data }
|
||||
when(resp_data_done) { state := s_outer_write }
|
||||
}
|
||||
is(s_outer_write) {
|
||||
io.outer.acquire.valid := Bool(true)
|
||||
when(outer_data_write_done) {
|
||||
when(macq_data_done) {
|
||||
io.wb.resp.valid := Bool(true)
|
||||
state := s_idle
|
||||
}
|
||||
@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa
|
||||
val wb = new L2WritebackIO
|
||||
}
|
||||
|
||||
val c_acq = io.inner.acquire.bits
|
||||
val c_rel = io.inner.release.bits
|
||||
val c_gnt = io.inner.grant.bits
|
||||
val c_ack = io.inner.finish.bits
|
||||
val m_gnt = io.outer.grant.bits
|
||||
|
||||
def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = {
|
||||
//TODO apply acq's write mask
|
||||
Mux(co.messageHasData(acq), old_data, new_data)
|
||||
}
|
||||
val cacq = io.inner.acquire.bits
|
||||
val crel = io.inner.release.bits
|
||||
val cgnt = io.inner.grant.bits
|
||||
val cack = io.inner.finish.bits
|
||||
val mgnt = io.outer.grant.bits
|
||||
}
|
||||
|
||||
class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
|
||||
val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7)
|
||||
val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7)
|
||||
val state = Reg(init=s_idle)
|
||||
|
||||
val xact_src = Reg(io.inner.release.bits.header.src.clone)
|
||||
val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
|
||||
val xact_addr = Reg(io.inner.release.bits.payload.addr.clone)
|
||||
val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone)
|
||||
val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone)
|
||||
val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
|
||||
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
|
||||
val xact_tag_match = Reg{ Bool() }
|
||||
val xact_meta = Reg{ new L2Metadata }
|
||||
val xact_way_en = Reg{ Bits(width = nWays) }
|
||||
val xact = Release(xact_r_type, xact_addr, xact_client_xact_id)
|
||||
val xact = Release(
|
||||
voluntary = Bool(true),
|
||||
r_type = xact_r_type,
|
||||
client_xact_id = xact_client_xact_id,
|
||||
addr_block = xact_addr_block)
|
||||
|
||||
val collect_inner_data = Reg(init=Bool(false))
|
||||
val (inner_data_cnt, inner_data_done) =
|
||||
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
|
||||
val (local_data_cnt, local_data_done) =
|
||||
Counter(io.data.write.fire(), tlDataBeats)
|
||||
val collect_crel_data = Reg(init=Bool(false))
|
||||
val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats))
|
||||
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
|
||||
val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write)
|
||||
|
||||
io.has_acquire_conflict := Bool(false)
|
||||
io.has_acquire_match := Bool(false)
|
||||
io.has_release_match := co.isVoluntary(c_rel.payload)
|
||||
io.has_release_match := crel.payload.isVoluntary()
|
||||
|
||||
io.outer.grant.ready := Bool(false)
|
||||
io.outer.acquire.valid := Bool(false)
|
||||
@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
|
||||
|
||||
io.inner.grant.bits.header.src := UInt(bankId)
|
||||
io.inner.grant.bits.header.dst := xact_src
|
||||
io.inner.grant.bits.payload := Grant(Bool(false),
|
||||
co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh),
|
||||
xact_client_xact_id,
|
||||
UInt(trackerId))
|
||||
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh)
|
||||
|
||||
io.data.read.valid := Bool(false)
|
||||
io.data.write.valid := Bool(false)
|
||||
io.data.write.bits.id := UInt(trackerId)
|
||||
io.data.write.bits.way_en := xact_way_en
|
||||
io.data.write.bits.addr := Cat(xact_addr, local_data_cnt)
|
||||
io.data.write.bits.addr_block := xact_addr_block
|
||||
io.data.write.bits.addr_beat := write_data_cnt
|
||||
io.data.write.bits.wmask := SInt(-1)
|
||||
io.data.write.bits.data := xact_data(local_data_cnt)
|
||||
io.data.write.bits.data := xact_data(write_data_cnt)
|
||||
io.meta.read.valid := Bool(false)
|
||||
io.meta.read.bits.id := UInt(trackerId)
|
||||
io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB)
|
||||
io.meta.read.bits.tag := xact_addr >> UInt(idxBits)
|
||||
io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB)
|
||||
io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits)
|
||||
io.meta.write.valid := Bool(false)
|
||||
io.meta.write.bits.id := UInt(trackerId)
|
||||
io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB)
|
||||
io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB)
|
||||
io.meta.write.bits.way_en := xact_way_en
|
||||
io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits)
|
||||
io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits)
|
||||
io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact,
|
||||
xact_meta.coh,
|
||||
xact_src)
|
||||
io.wb.req.valid := Bool(false)
|
||||
|
||||
when(collect_inner_data) {
|
||||
when(collect_crel_data) {
|
||||
io.inner.release.ready := Bool(true)
|
||||
when(io.inner.release.valid) {
|
||||
xact_data(inner_data_cnt) := c_rel.payload.data
|
||||
xact_data(crel.payload.addr_beat) := crel.payload.data
|
||||
crel_data_valid(crel.payload.addr_beat) := Bool(true)
|
||||
}
|
||||
when(inner_data_done) { collect_inner_data := Bool(false) }
|
||||
when(crel_data_done) { collect_crel_data := Bool(false) }
|
||||
}
|
||||
|
||||
switch (state) {
|
||||
is(s_idle) {
|
||||
io.inner.release.ready := Bool(true)
|
||||
when( io.inner.release.valid ) {
|
||||
xact_src := c_rel.header.src
|
||||
xact_r_type := c_rel.payload.r_type
|
||||
xact_addr := c_rel.payload.addr
|
||||
xact_client_xact_id := c_rel.payload.client_xact_id
|
||||
xact_data(UInt(0)) := c_rel.payload.data
|
||||
collect_inner_data := co.messageHasData(c_rel.payload)
|
||||
xact_src := crel.header.src
|
||||
xact_r_type := crel.payload.r_type
|
||||
xact_addr_block := crel.payload.addr_block
|
||||
xact_addr_beat := crel.payload.addr_beat
|
||||
xact_client_xact_id := crel.payload.client_xact_id
|
||||
xact_data(UInt(0)) := crel.payload.data
|
||||
collect_crel_data := crel.payload.hasMultibeatData()
|
||||
crel_data_valid := Bits(1)
|
||||
state := s_meta_read
|
||||
}
|
||||
}
|
||||
@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
|
||||
xact_meta := io.meta.resp.bits.meta
|
||||
xact_way_en := io.meta.resp.bits.way_en
|
||||
state := Mux(io.meta.resp.bits.tag_match,
|
||||
Mux(co.messageHasData(xact), s_data_write, s_meta_write),
|
||||
s_grant)
|
||||
Mux(xact.hasData(), s_data_write, s_meta_write),
|
||||
Mux(xact.requiresAck(), s_grant, s_idle))
|
||||
}
|
||||
}
|
||||
is(s_data_write) {
|
||||
io.data.write.valid := (if(tlDataBeats == 1) Bool(true)
|
||||
else !collect_inner_data || (local_data_cnt < inner_data_cnt))
|
||||
when(local_data_done) { state := s_meta_write }
|
||||
io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt)
|
||||
when(write_data_done) { state := s_meta_write }
|
||||
}
|
||||
is(s_meta_write) {
|
||||
io.meta.write.valid := Bool(true)
|
||||
when(io.meta.write.ready) { state := s_grant }
|
||||
when(io.meta.write.ready) {
|
||||
state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck?
|
||||
}
|
||||
}
|
||||
is(s_grant) {
|
||||
io.inner.grant.valid := Bool(true)
|
||||
when(io.inner.grant.ready) {
|
||||
state := Mux(co.requiresAckForGrant(c_gnt.payload),
|
||||
s_busy, s_idle)
|
||||
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
|
||||
}
|
||||
}
|
||||
is(s_busy) {
|
||||
is(s_ack) {
|
||||
// TODO: This state is unnecessary if no client will ever issue the
|
||||
// pending Acquire that caused this writeback until it receives the
|
||||
// Grant.voluntaryAck for this writeback
|
||||
io.inner.finish.ready := Bool(true)
|
||||
when(io.inner.finish.valid) { state := s_idle }
|
||||
}
|
||||
@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
|
||||
|
||||
|
||||
class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
|
||||
val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14)
|
||||
val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14)
|
||||
val state = Reg(init=s_idle)
|
||||
|
||||
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
|
||||
val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone)
|
||||
val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
|
||||
val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
|
||||
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
|
||||
val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone)
|
||||
val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
|
||||
val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
|
||||
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
|
||||
val xact_tag_match = Reg{ Bool() }
|
||||
val xact_meta = Reg{ new L2Metadata }
|
||||
val xact_way_en = Reg{ Bits(width = nWays) }
|
||||
val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock)
|
||||
val xact = Acquire(
|
||||
uncached = xact_uncached,
|
||||
a_type = xact_a_type,
|
||||
client_xact_id = xact_client_xact_id,
|
||||
addr_block = xact_addr_block,
|
||||
addr_beat = xact_addr_beat,
|
||||
data = UInt(0),
|
||||
subblock = xact_subblock)
|
||||
|
||||
val collect_cacq_data = Reg(init=Bool(false))
|
||||
val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats))
|
||||
val crel_had_data = Reg(init = Bool(false))
|
||||
val release_count = Reg(init = UInt(0, width = log2Up(nClients+1)))
|
||||
val pending_probes = Reg(init = UInt(0, width = nCoherentClients))
|
||||
val curr_p_id = co.dir.next(pending_probes)
|
||||
val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers)
|
||||
val mask_self = Mux(co.requiresSelfProbe(xact),
|
||||
val mask_self = Mux(xact.requiresSelfProbe(),
|
||||
full_sharers | (UInt(1) << xact_src),
|
||||
full_sharers & ~UInt(UInt(1) << xact_src, width = nClients))
|
||||
val mask_incoherent = mask_self & ~io.tile_incoherent
|
||||
|
||||
val collect_cacq_data = Reg(init=Bool(false))
|
||||
//TODO: zero width wires
|
||||
val (cacq_data_cnt, cacq_data_done) =
|
||||
Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats)
|
||||
val (crel_data_cnt, crel_data_done) =
|
||||
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
|
||||
val (cgnt_data_cnt, cgnt_data_done) =
|
||||
Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats)
|
||||
val (outer_data_write_cnt, outer_data_write_done) =
|
||||
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
|
||||
val (outer_data_resp_cnt, outer_data_resp_done) =
|
||||
Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats)
|
||||
val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats)
|
||||
val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats)
|
||||
val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats)
|
||||
val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire)
|
||||
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
|
||||
val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
|
||||
val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
|
||||
val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1))
|
||||
val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1))
|
||||
val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
|
||||
|
||||
val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh)
|
||||
val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit
|
||||
val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh)
|
||||
val needs_probes = co.requiresProbes(xact, xact_meta.coh)
|
||||
//TODO: uncached does or does not allocate
|
||||
//val do_allocate = !xact_uncached || xact.allocate()
|
||||
|
||||
val amoalu = Module(new AMOALU)
|
||||
amoalu.io.addr := xact.addr()
|
||||
amoalu.io.cmd := xact.op_code()
|
||||
amoalu.io.typ := xact.op_size()
|
||||
amoalu.io.lhs := io.data.resp.bits.data //default
|
||||
amoalu.io.rhs := xact.data(0) // default
|
||||
|
||||
def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) {
|
||||
val old_data = incoming.data
|
||||
val new_data = buffer(incoming.addr_beat)
|
||||
amoalu.io.lhs := old_data
|
||||
amoalu.io.rhs := new_data
|
||||
val wmask = FillInterleaved(8, xact.write_mask())
|
||||
buffer(incoming.addr_beat) :=
|
||||
Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out,
|
||||
Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite),
|
||||
wmask & new_data | ~wmask & old_data, old_data))
|
||||
}
|
||||
|
||||
//TODO: Are there any races between lines with the same idx?
|
||||
//TODO: Allow hit under miss for stores
|
||||
io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) ||
|
||||
xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) &&
|
||||
io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) ||
|
||||
xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) &&
|
||||
(state != s_idle) &&
|
||||
!collect_cacq_data
|
||||
io.has_acquire_match := co.messageHasData(xact) &&
|
||||
(xact.addr === c_acq.payload.addr) &&
|
||||
io.has_acquire_match := xact.hasMultibeatData() &&
|
||||
(xact.addr_block === cacq.payload.addr_block) &&
|
||||
collect_cacq_data
|
||||
io.has_release_match := !co.isVoluntary(c_rel.payload) &&
|
||||
(xact.addr === c_rel.payload.addr) &&
|
||||
io.has_release_match := !crel.payload.isVoluntary() &&
|
||||
(xact.addr_block === crel.payload.addr_block) &&
|
||||
(state === s_probe)
|
||||
|
||||
val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src)
|
||||
val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh,
|
||||
c_gnt.header.dst)
|
||||
val next_coh_on_rel = co.managerMetadataOnRelease(
|
||||
incoming = crel.payload,
|
||||
meta = xact_meta.coh,
|
||||
src = crel.header.src)
|
||||
val next_coh_on_gnt = co.managerMetadataOnGrant(
|
||||
outgoing = cgnt.payload,
|
||||
meta = xact_meta.coh,
|
||||
dst = cgnt.header.dst)
|
||||
|
||||
val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)),
|
||||
{ case TLId => outerId })
|
||||
val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId })
|
||||
val outer_write = Bundle(UncachedWriteBlock(
|
||||
client_xact_id = UInt(trackerId),
|
||||
addr_block = xact_addr_block,
|
||||
addr_beat = macq_data_cnt,
|
||||
data = xact_data(macq_data_cnt)),
|
||||
{ case TLId => outerId })
|
||||
val outer_read = Bundle(UncachedReadBlock(
|
||||
client_xact_id = UInt(trackerId),
|
||||
addr_block = xact_addr_block),
|
||||
{ case TLId => outerId })
|
||||
|
||||
io.outer.acquire.valid := Bool(false)
|
||||
io.outer.acquire.bits.payload := outer_read //default
|
||||
@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
io.inner.probe.valid := Bool(false)
|
||||
io.inner.probe.bits.header.src := UInt(bankId)
|
||||
io.inner.probe.bits.header.dst := curr_p_id
|
||||
io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr)
|
||||
io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh)
|
||||
|
||||
io.inner.grant.valid := Bool(false)
|
||||
io.inner.grant.bits.header.src := UInt(bankId)
|
||||
io.inner.grant.bits.header.dst := xact_src
|
||||
io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh),
|
||||
xact_client_xact_id,
|
||||
UInt(trackerId),
|
||||
xact_data(cgnt_data_cnt))
|
||||
io.inner.grant.bits.payload := xact.makeGrant(
|
||||
manager_xact_id = UInt(trackerId),
|
||||
meta = xact_meta.coh,
|
||||
addr_beat = cgnt_data_cnt,
|
||||
data = xact_data(cgnt_data_cnt))
|
||||
|
||||
io.inner.acquire.ready := Bool(false)
|
||||
io.inner.release.ready := Bool(false)
|
||||
@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
io.data.read.valid := Bool(false)
|
||||
io.data.read.bits.id := UInt(trackerId)
|
||||
io.data.read.bits.way_en := xact_way_en
|
||||
io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt)
|
||||
io.data.read.bits.addr_block := xact_addr_block
|
||||
io.data.read.bits.addr_beat := read_data_cnt
|
||||
io.data.write.valid := Bool(false)
|
||||
io.data.write.bits.id := UInt(trackerId)
|
||||
io.data.write.bits.way_en := xact_way_en
|
||||
io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt)
|
||||
io.data.write.bits.addr_block := xact_addr_block
|
||||
io.data.write.bits.addr_beat := write_data_cnt
|
||||
io.data.write.bits.wmask := SInt(-1)
|
||||
io.data.write.bits.data := xact_data(local_data_write_cnt)
|
||||
io.data.write.bits.data := xact_data(write_data_cnt)
|
||||
io.meta.read.valid := Bool(false)
|
||||
io.meta.read.bits.id := UInt(trackerId)
|
||||
io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB)
|
||||
io.meta.read.bits.tag := xact_addr >> UInt(idxBits)
|
||||
io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB)
|
||||
io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits)
|
||||
io.meta.write.valid := Bool(false)
|
||||
io.meta.write.bits.id := UInt(trackerId)
|
||||
io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB)
|
||||
io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB)
|
||||
io.meta.write.bits.way_en := xact_way_en
|
||||
io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits)
|
||||
io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits)
|
||||
io.meta.write.bits.data.coh := next_coh_on_gnt
|
||||
|
||||
io.wb.req.valid := Bool(false)
|
||||
io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB))
|
||||
io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB))
|
||||
io.wb.req.bits.coh := xact_meta.coh
|
||||
io.wb.req.bits.way_en := xact_way_en
|
||||
io.wb.req.bits.id := UInt(trackerId)
|
||||
@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
when(collect_cacq_data) {
|
||||
io.inner.acquire.ready := Bool(true)
|
||||
when(io.inner.acquire.valid) {
|
||||
xact_data(cacq_data_cnt) := c_acq.payload.data
|
||||
xact_data(cacq.payload.addr_beat) := cacq.payload.data
|
||||
cacq_data_valid(cacq.payload.addr_beat) := Bool(true)
|
||||
}
|
||||
when(cacq_data_done) { collect_cacq_data := Bool(false) }
|
||||
}
|
||||
@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
is(s_idle) {
|
||||
io.inner.acquire.ready := Bool(true)
|
||||
when( io.inner.acquire.valid ) {
|
||||
xact_uncached := c_acq.payload.uncached
|
||||
xact_a_type := c_acq.payload.a_type
|
||||
xact_addr := c_acq.payload.addr
|
||||
xact_client_xact_id := c_acq.payload.client_xact_id
|
||||
xact_data(UInt(0)) := c_acq.payload.data
|
||||
xact_subblock := c_acq.payload.subblock
|
||||
xact_src := c_acq.header.src
|
||||
collect_cacq_data := co.messageHasData(c_acq.payload)
|
||||
xact_uncached := cacq.payload.uncached
|
||||
xact_a_type := cacq.payload.a_type
|
||||
xact_addr_block := cacq.payload.addr_block
|
||||
xact_addr_beat := cacq.payload.addr_beat
|
||||
xact_client_xact_id := cacq.payload.client_xact_id
|
||||
xact_data(UInt(0)) := cacq.payload.data
|
||||
xact_subblock := cacq.payload.subblock
|
||||
xact_src := cacq.header.src
|
||||
collect_cacq_data := cacq.payload.hasMultibeatData()
|
||||
state := s_meta_read
|
||||
}
|
||||
}
|
||||
@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
xact_way_en := io.meta.resp.bits.way_en
|
||||
val coh = io.meta.resp.bits.meta.coh
|
||||
val _tag_match = io.meta.resp.bits.tag_match
|
||||
val _needs_writeback = !_tag_match && co.needsWriteback(coh)
|
||||
val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit
|
||||
val _needs_probes = _tag_match && co.requiresProbes(xact, coh)
|
||||
val _is_hit = _tag_match && co.isHit(xact, coh)
|
||||
val full_block = !xact.uncached ||
|
||||
xact.is(Acquire.uncachedReadBlock) ||
|
||||
xact.is(Acquire.uncachedWriteBlock)
|
||||
read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
|
||||
read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat)
|
||||
write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat)
|
||||
write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat)
|
||||
resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
|
||||
resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat)
|
||||
cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
|
||||
cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat)
|
||||
when(_needs_probes) {
|
||||
pending_probes := mask_incoherent(nCoherentClients-1,0)
|
||||
release_count := co.dir.count(mask_incoherent)
|
||||
@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
pending_probes := co.dir.pop(pending_probes, curr_p_id)
|
||||
}
|
||||
// Handle releases, which may have data being written back
|
||||
//TODO: make sure cacq data is actually present before accpeting
|
||||
// release data to merge!
|
||||
io.inner.release.ready := Bool(true)
|
||||
when(io.inner.release.valid) {
|
||||
xact_meta.coh := next_coh_on_rel
|
||||
// Handle released dirty data
|
||||
when(co.messageHasData(c_rel.payload)) {
|
||||
when(crel.payload.hasData()) {
|
||||
crel_had_data := Bool(true)
|
||||
//TODO make sure cacq data is actually present before merging
|
||||
xact_data(crel_data_cnt) := mergeData(xact,
|
||||
xact_data(crel_data_cnt),
|
||||
c_rel.payload.data)
|
||||
mergeData(xact_data, crel.payload)
|
||||
}
|
||||
// We don't decrement release_count until we've received all the data beats.
|
||||
when(!co.messageHasData(c_rel.payload) || crel_data_done) {
|
||||
when(!crel.payload.hasMultibeatData() || crel_data_done) {
|
||||
release_count := release_count - UInt(1)
|
||||
}
|
||||
}
|
||||
@ -847,37 +999,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
io.outer.grant.ready := Bool(true)
|
||||
when(io.outer.grant.valid) {
|
||||
//TODO make sure cacq data is actually present before merging
|
||||
xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt),
|
||||
io.outer.grant.bits.payload.data)
|
||||
//TODO: set pending client state in xact_meta.coh
|
||||
when(outer_data_resp_done) {
|
||||
state := Mux(co.messageHasData(io.outer.grant.bits.payload),
|
||||
s_data_write, s_data_read)
|
||||
mergeData(xact_data, mgnt.payload)
|
||||
when(mgnt_data_done) {
|
||||
state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read)
|
||||
}
|
||||
}
|
||||
}
|
||||
is(s_data_read) {
|
||||
io.data.read.valid := (if(tlDataBeats == 1) Bool(true)
|
||||
else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt))
|
||||
io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt)
|
||||
when(io.data.resp.valid) {
|
||||
xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt),
|
||||
io.data.resp.bits.data)
|
||||
mergeData(xact_data, io.data.resp.bits)
|
||||
resp_data_cnt := resp_data_cnt + UInt(1)
|
||||
}
|
||||
when(io.data.read.ready) {
|
||||
read_data_cnt := read_data_cnt + UInt(1)
|
||||
when(read_data_cnt === read_data_max) { state := s_data_resp }
|
||||
}
|
||||
when(local_data_read_done) { state := s_data_resp }
|
||||
}
|
||||
is(s_data_resp) {
|
||||
when(io.data.resp.valid) {
|
||||
xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt),
|
||||
io.data.resp.bits.data)
|
||||
mergeData(xact_data, io.data.resp.bits)
|
||||
resp_data_cnt := resp_data_cnt + UInt(1)
|
||||
}
|
||||
when(local_data_resp_done) {
|
||||
state := Mux(co.messageHasData(xact), s_data_write, s_meta_write)
|
||||
when(resp_data_cnt === resp_data_max) {
|
||||
state := Mux(xact.hasData(), s_data_write, s_meta_write)
|
||||
}
|
||||
}
|
||||
is(s_data_write) {
|
||||
io.data.write.valid := Bool(true)
|
||||
when(local_data_write_done) {
|
||||
state := s_meta_write
|
||||
when(io.data.write.ready) {
|
||||
write_data_cnt := write_data_cnt + UInt(1)
|
||||
when(write_data_cnt === write_data_max) {
|
||||
state := s_meta_write
|
||||
}
|
||||
}
|
||||
}
|
||||
is(s_meta_write) {
|
||||
@ -886,12 +1040,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
|
||||
}
|
||||
is(s_grant) {
|
||||
io.inner.grant.valid := Bool(true)
|
||||
when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) {
|
||||
state := Mux(co.requiresAckForGrant(c_gnt.payload),
|
||||
s_busy, s_idle)
|
||||
when(io.inner.grant.ready) {
|
||||
cgnt_data_cnt := cgnt_data_cnt + UInt(1)
|
||||
when(cgnt_data_cnt === cgnt_data_max) {
|
||||
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
|
||||
}
|
||||
}
|
||||
}
|
||||
is(s_busy) {
|
||||
is(s_ack) {
|
||||
io.inner.finish.ready := Bool(true)
|
||||
when(io.inner.finish.valid) { state := s_idle }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user