riscv/rocket-chip
1
0
Fork 0
Code Releases Activity

Support for uncached sub-block reads and writes, major TileLink and CoherencePolicy refactor.

Browse Source
This commit is contained in:
Henry Cook 2015-02-01 19:57:53 -08:00
parent 7b4e9dd137
commit 3aa030f960
8 changed files with 1343 additions and 1014 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

526
uncore/src/main/scala/cache.scala
View File

@ -2,32 +2,26 @@
package uncore
import Chisel._
import scala.reflect.ClassTag
case object CacheName extends Field[String]
case object NSets extends Field[Int]
case object NWays extends Field[Int]
case object BlockOffBits extends Field[Int]
case object RowBits extends Field[Int]
case object WordBits extends Field[Int]
case object Replacer extends Field[() => ReplacementPolicy]
case object AmoAluOperandBits extends Field[Int]
abstract trait CacheParameters extends UsesParameters {
val paddrBits = params(PAddrBits)
val vaddrBits = params(VAddrBits)
val pgIdxBits = params(PgIdxBits)
val nSets = params(NSets)
val blockOffBits = params(BlockOffBits)
val idxBits = log2Up(nSets)
val untagBits = blockOffBits + idxBits
val tagBits = paddrBits - untagBits
val tagBits = params(PAddrBits) - untagBits
val nWays = params(NWays)
val wayBits = log2Up(nWays)
val isDM = nWays == 1
val wordBits = params(WordBits)
val wordBytes = wordBits/8
val wordOffBits = log2Up(wordBytes)
val rowBits = params(RowBits)
val rowWords = rowBits/wordBits
val rowBytes = rowBits/8
val rowOffBits = log2Up(rowBytes)
}
@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters {
abstract class CacheBundle extends Bundle with CacheParameters
abstract class CacheModule extends Module with CacheParameters
class StoreGen(typ: Bits, addr: Bits, dat: Bits) {
val byte = typ === MT_B || typ === MT_BU
val half = typ === MT_H || typ === MT_HU
val word = typ === MT_W || typ === MT_WU
def mask =
Mux(byte, Bits( 1) << addr(2,0),
Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)),
Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)),
Bits(255))))
def data =
Mux(byte, Fill(8, dat( 7,0)),
Mux(half, Fill(4, dat(15,0)),
wordData))
lazy val wordData =
Mux(word, Fill(2, dat(31,0)),
dat)
}
class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) {
val t = new StoreGen(typ, addr, dat)
val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D
val wordShift = Mux(addr(2), dat(63,32), dat(31,0))
val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift)
val halfShift = Mux(addr(1), word(31,16), word(15,0))
val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift)
val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0)))
val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift)
}
class AMOALU extends CacheModule {
val operandBits = params(AmoAluOperandBits)
require(operandBits == 64)
val io = new Bundle {
val addr = Bits(INPUT, blockOffBits)
val cmd = Bits(INPUT, M_SZ)
val typ = Bits(INPUT, MT_SZ)
val lhs = Bits(INPUT, operandBits)
val rhs = Bits(INPUT, operandBits)
val out = Bits(OUTPUT, operandBits)
}
val storegen = new StoreGen(io.typ, io.addr, io.rhs)
val rhs = storegen.wordData
val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX
val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU
val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU
val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization:
io.typ === MT_B || io.typ === MT_BU
val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31))
val adder_out = (io.lhs & mask).toUInt + (rhs & mask)
val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63))
val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63))
val lt_lo = io.lhs(31,0) < rhs(31,0)
val lt_hi = io.lhs(63,32) < rhs(63,32)
val eq_hi = io.lhs(63,32) === rhs(63,32)
val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo)
val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs))
val out = Mux(io.cmd === M_XA_ADD, adder_out,
Mux(io.cmd === M_XA_AND, io.lhs & rhs,
Mux(io.cmd === M_XA_OR, io.lhs | rhs,
Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs,
Mux(Mux(less, min, max), io.lhs,
storegen.data)))))
val wmask = FillInterleaved(8, storegen.mask)
io.out := wmask & out | ~wmask & io.lhs
}
abstract class ReplacementPolicy {
def way: UInt
def miss: Unit
@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule {
io.write.ready := !rst
}
abstract trait L2HellaCacheParameters extends CacheParameters
with CoherenceAgentParameters {
abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters {
val idxMSB = idxBits-1
val idxLSB = 0
val refillCyclesPerBeat = params(TLDataBits)/rowBits
val refillCycles = refillCyclesPerBeat*params(TLDataBeats)
require(refillCyclesPerBeat == 1)
}
abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters
abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters
abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters
abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters {
def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = {
val (cnt, cnt_done) =
Counter(inc && data.hasMultibeatData(), tlDataBeats)
val done = (inc && !data.hasMultibeatData()) || cnt_done
(cnt, done)
}
def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = {
connectDataBeatCounter(in.fire(), in.bits.payload)
}
def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = {
connectDataBeatCounter(in.fire(), in.bits.payload)._2
}
def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = {
connectDataBeatCounter(in.fire(), in.bits)
}
def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = {
connectDataBeatCounter(in.valid, in.bits)._2
}
}
trait HasL2Id extends Bundle with CoherenceAgentParameters {
val id = UInt(width = log2Up(nTransactors + 1))
@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle {
val way_en = Bits(width = nWays)
}
trait HasL2Data extends HasTileLinkData {
def hasData(dummy: Int = 0) = Bool(true)
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1)
}
object L2Metadata {
def apply(tag: Bits, coh: ManagerMetadata) = {
val meta = new L2Metadata
@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule {
io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en)
}
class L2DataReadReq extends L2HellaCacheBundle with HasL2Id {
class L2DataReadReq extends L2HellaCacheBundle
with HasCacheBlockAddress
with HasTileLinkBeatId
with HasL2Id {
val way_en = Bits(width = nWays)
val addr = Bits(width = tlAddrBits)
}
class L2DataWriteReq extends L2DataReadReq {
class L2DataWriteReq extends L2DataReadReq
with HasL2Data {
val wmask = Bits(width = tlWriteMaskBits)
val data = Bits(width = tlDataBits)
}
class L2DataResp extends Bundle with HasL2Id with TileLinkParameters {
val data = Bits(width = tlDataBits)
}
class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data
trait HasL2DataReadIO extends L2HellaCacheBundle {
val read = Decoupled(new L2DataReadReq)
@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule {
val wmask = FillInterleaved(8, io.write.bits.wmask)
val reg_raddr = Reg(UInt())
val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true)
val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr)
val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr)
val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat)
val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat)
when (io.write.bits.way_en.orR && io.write.valid) {
array.write(waddr, io.write.bits.data, wmask)
@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule {
io.resp.valid := ShiftRegister(io.read.fire(), 1)
io.resp.bits.id := ShiftRegister(io.read.bits.id, 1)
io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1)
io.resp.bits.data := array(reg_raddr)
io.read.ready := !io.write.valid
io.write.ready := Bool(true)
@ -261,7 +353,8 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
}
// Wiring helper funcs
def doOutputArbitration[T <: Data](out: DecoupledIO[T],
def doOutputArbitration[T <: Data](
out: DecoupledIO[T],
ins: Seq[DecoupledIO[T]],
count: Int = 1,
lock: T => Bool = (a: T) => Bool(true)) {
@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe)
// Wire grant reply to initiating client
def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload)
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _)
doOutputArbitration(
io.inner.grant,
trackerList.map(_.io.inner.grant),
tlDataBeats,
(m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData())
// Create an arbiter for the one memory port
val outerList = trackerList.map(_.io.outer) :+ wb.io.outer
@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
// Wire local memories
doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read))
doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write))
doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats)
doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats)
doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read)
doOutputArbitration(io.data.write, trackerList.map(_.io.data.write))
doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp))
doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp)
}
class L2WritebackReq extends L2HellaCacheBundle
with HasL2Id {
val addr = UInt(width = tlAddrBits)
val addr_block = UInt(width = tlBlockAddrBits)
val coh = new ManagerMetadata
val way_en = Bits(width = nWays)
}
@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
val has_release_match = Bool(OUTPUT)
val data = new L2DataRWIO
}
val c_acq = io.inner.acquire.bits
val c_rel = io.inner.release.bits
val c_gnt = io.inner.grant.bits
val cacq = io.inner.acquire.bits
val crel = io.inner.release.bits
val cgnt = io.inner.grant.bits
val c_ack = io.inner.finish.bits
val m_gnt = io.outer.grant.bits
val mgnt = io.outer.grant.bits
val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5)
val state = Reg(init=s_idle)
val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
val xact_coh = Reg{ new ManagerMetadata }
val xact_way_en = Reg{ Bits(width = nWays) }
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
val pending_probes = Reg(init = co.dir.flush)
val curr_p_id = co.dir.next(pending_probes)
val (crel_data_cnt, crel_data_done) =
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
val (outer_data_write_cnt, outer_data_write_done) =
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats)
val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats)
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats)
val resp_data_done = connectIncomingDataBeatCounter(io.data.resp)
io.has_release_match := !co.isVoluntary(c_rel.payload) &&
co.isCoherenceConflict(xact_addr, c_rel.payload.addr) &&
io.has_release_match := !crel.payload.isVoluntary() &&
co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) &&
(state === s_probe)
val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src)
val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src)
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr,
UInt(trackerId),
xact_data(outer_data_write_cnt)),
io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = macq_data_cnt,
data = xact_data(macq_data_cnt)),
{ case TLId => outerId })
io.outer.grant.ready := Bool(false) // Never gets mgnts
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr)
io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block)
io.inner.grant.valid := Bool(false)
io.inner.acquire.ready := Bool(false)
@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
io.data.read.valid := Bool(false)
io.data.read.bits.id := UInt(trackerId)
io.data.read.bits.way_en := xact_way_en
io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt)
io.data.read.bits.addr_block := xact_addr_block
io.data.read.bits.addr_beat := read_data_cnt
io.data.write.valid := Bool(false)
io.wb.req.ready := Bool(false)
@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
is(s_idle) {
io.wb.req.ready := Bool(true)
when(io.wb.req.valid) {
xact_addr := io.wb.req.bits.addr
xact_addr_block := io.wb.req.bits.addr_block
xact_coh := io.wb.req.bits.coh
xact_way_en := io.wb.req.bits.way_en
xact_id := io.wb.req.bits.id
@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
when(io.inner.release.valid) {
xact_coh := next_coh_on_rel
// Handle released dirty data
when(co.messageHasData(c_rel.payload)) {
when(crel.payload.hasData()) {
crel_had_data := Bool(true)
xact_data(crel_data_cnt) := c_rel.payload.data
xact_data(crel.payload.addr_beat) := crel.payload.data
}
// We don't decrement release_count until we've received all the data beats.
when(!co.messageHasData(c_rel.payload) || crel_data_done) {
when(!crel.payload.hasData() || crel_data_done) {
release_count := release_count - UInt(1)
}
}
@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str
}
is(s_data_read) {
io.data.read.valid := Bool(true)
when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data }
when(local_data_read_done) { state := s_data_resp }
when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data }
when(read_data_done) { state := s_data_resp }
}
is(s_data_resp) {
when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data }
when(local_data_resp_done) { state := s_outer_write }
when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data }
when(resp_data_done) { state := s_outer_write }
}
is(s_outer_write) {
io.outer.acquire.valid := Bool(true)
when(outer_data_write_done) {
when(macq_data_done) {
io.wb.resp.valid := Bool(true)
state := s_idle
}
@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa
val wb = new L2WritebackIO
}
val c_acq = io.inner.acquire.bits
val c_rel = io.inner.release.bits
val c_gnt = io.inner.grant.bits
val c_ack = io.inner.finish.bits
val m_gnt = io.outer.grant.bits
def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = {
//TODO apply acq's write mask
Mux(co.messageHasData(acq), old_data, new_data)
}
val cacq = io.inner.acquire.bits
val crel = io.inner.release.bits
val cgnt = io.inner.grant.bits
val cack = io.inner.finish.bits
val mgnt = io.outer.grant.bits
}
class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7)
val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.release.bits.header.src.clone)
val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
val xact_addr = Reg(io.inner.release.bits.payload.addr.clone)
val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone)
val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone)
val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
val xact_tag_match = Reg{ Bool() }
val xact_meta = Reg{ new L2Metadata }
val xact_way_en = Reg{ Bits(width = nWays) }
val xact = Release(xact_r_type, xact_addr, xact_client_xact_id)
val xact = Release(
voluntary = Bool(true),
r_type = xact_r_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block)
val collect_inner_data = Reg(init=Bool(false))
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
val (local_data_cnt, local_data_done) =
Counter(io.data.write.fire(), tlDataBeats)
val collect_crel_data = Reg(init=Bool(false))
val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats))
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write)
io.has_acquire_conflict := Bool(false)
io.has_acquire_match := Bool(false)
io.has_release_match := co.isVoluntary(c_rel.payload)
io.has_release_match := crel.payload.isVoluntary()
io.outer.grant.ready := Bool(false)
io.outer.acquire.valid := Bool(false)
@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := Grant(Bool(false),
co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh),
xact_client_xact_id,
UInt(trackerId))
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh)
io.data.read.valid := Bool(false)
io.data.write.valid := Bool(false)
io.data.write.bits.id := UInt(trackerId)
io.data.write.bits.way_en := xact_way_en
io.data.write.bits.addr := Cat(xact_addr, local_data_cnt)
io.data.write.bits.addr_block := xact_addr_block
io.data.write.bits.addr_beat := write_data_cnt
io.data.write.bits.wmask := SInt(-1)
io.data.write.bits.data := xact_data(local_data_cnt)
io.data.write.bits.data := xact_data(write_data_cnt)
io.meta.read.valid := Bool(false)
io.meta.read.bits.id := UInt(trackerId)
io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB)
io.meta.read.bits.tag := xact_addr >> UInt(idxBits)
io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB)
io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits)
io.meta.write.valid := Bool(false)
io.meta.write.bits.id := UInt(trackerId)
io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB)
io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB)
io.meta.write.bits.way_en := xact_way_en
io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits)
io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits)
io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact,
xact_meta.coh,
xact_src)
io.wb.req.valid := Bool(false)
when(collect_inner_data) {
when(collect_crel_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
xact_data(inner_data_cnt) := c_rel.payload.data
xact_data(crel.payload.addr_beat) := crel.payload.data
crel_data_valid(crel.payload.addr_beat) := Bool(true)
}
when(inner_data_done) { collect_inner_data := Bool(false) }
when(crel_data_done) { collect_crel_data := Bool(false) }
}
switch (state) {
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := c_rel.header.src
xact_r_type := c_rel.payload.r_type
xact_addr := c_rel.payload.addr
xact_client_xact_id := c_rel.payload.client_xact_id
xact_data(UInt(0)) := c_rel.payload.data
collect_inner_data := co.messageHasData(c_rel.payload)
xact_src := crel.header.src
xact_r_type := crel.payload.r_type
xact_addr_block := crel.payload.addr_block
xact_addr_beat := crel.payload.addr_beat
xact_client_xact_id := crel.payload.client_xact_id
xact_data(UInt(0)) := crel.payload.data
collect_crel_data := crel.payload.hasMultibeatData()
crel_data_valid := Bits(1)
state := s_meta_read
}
}
@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
xact_meta := io.meta.resp.bits.meta
xact_way_en := io.meta.resp.bits.way_en
state := Mux(io.meta.resp.bits.tag_match,
Mux(co.messageHasData(xact), s_data_write, s_meta_write),
s_grant)
Mux(xact.hasData(), s_data_write, s_meta_write),
Mux(xact.requiresAck(), s_grant, s_idle))
}
}
is(s_data_write) {
io.data.write.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (local_data_cnt < inner_data_cnt))
when(local_data_done) { state := s_meta_write }
io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt)
when(write_data_done) { state := s_meta_write }
}
is(s_meta_write) {
io.meta.write.valid := Bool(true)
when(io.meta.write.ready) { state := s_grant }
when(io.meta.write.ready) {
state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck?
}
}
is(s_grant) {
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(co.requiresAckForGrant(c_gnt.payload),
s_busy, s_idle)
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_busy) {
is(s_ack) {
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14)
val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone)
val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone)
val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
val xact_tag_match = Reg{ Bool() }
val xact_meta = Reg{ new L2Metadata }
val xact_way_en = Reg{ Bits(width = nWays) }
val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock)
val xact = Acquire(
uncached = xact_uncached,
a_type = xact_a_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block,
addr_beat = xact_addr_beat,
data = UInt(0),
subblock = xact_subblock)
val collect_cacq_data = Reg(init=Bool(false))
val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats))
val crel_had_data = Reg(init = Bool(false))
val release_count = Reg(init = UInt(0, width = log2Up(nClients+1)))
val pending_probes = Reg(init = UInt(0, width = nCoherentClients))
val curr_p_id = co.dir.next(pending_probes)
val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers)
val mask_self = Mux(co.requiresSelfProbe(xact),
val mask_self = Mux(xact.requiresSelfProbe(),
full_sharers | (UInt(1) << xact_src),
full_sharers & ~UInt(UInt(1) << xact_src, width = nClients))
val mask_incoherent = mask_self & ~io.tile_incoherent
val collect_cacq_data = Reg(init=Bool(false))
//TODO: zero width wires
val (cacq_data_cnt, cacq_data_done) =
Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats)
val (crel_data_cnt, crel_data_done) =
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
val (cgnt_data_cnt, cgnt_data_done) =
Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats)
val (outer_data_write_cnt, outer_data_write_done) =
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
val (outer_data_resp_cnt, outer_data_resp_done) =
Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats)
val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats)
val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats)
val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats)
val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire)
val crel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1))
val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1))
val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1))
val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh)
val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit
val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh)
val needs_probes = co.requiresProbes(xact, xact_meta.coh)
//TODO: uncached does or does not allocate
//val do_allocate = !xact_uncached || xact.allocate()
val amoalu = Module(new AMOALU)
amoalu.io.addr := xact.addr()
amoalu.io.cmd := xact.op_code()
amoalu.io.typ := xact.op_size()
amoalu.io.lhs := io.data.resp.bits.data //default
amoalu.io.rhs := xact.data(0) // default
def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) {
val old_data = incoming.data
val new_data = buffer(incoming.addr_beat)
amoalu.io.lhs := old_data
amoalu.io.rhs := new_data
val wmask = FillInterleaved(8, xact.write_mask())
buffer(incoming.addr_beat) :=
Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out,
Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite),
wmask & new_data | ~wmask & old_data, old_data))
}
//TODO: Are there any races between lines with the same idx?
//TODO: Allow hit under miss for stores
io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) ||
xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) &&
io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) ||
xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) &&
(state != s_idle) &&
!collect_cacq_data
io.has_acquire_match := co.messageHasData(xact) &&
(xact.addr === c_acq.payload.addr) &&
io.has_acquire_match := xact.hasMultibeatData() &&
(xact.addr_block === cacq.payload.addr_block) &&
collect_cacq_data
io.has_release_match := !co.isVoluntary(c_rel.payload) &&
(xact.addr === c_rel.payload.addr) &&
io.has_release_match := !crel.payload.isVoluntary() &&
(xact.addr_block === crel.payload.addr_block) &&
(state === s_probe)
val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src)
val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh,
c_gnt.header.dst)
val next_coh_on_rel = co.managerMetadataOnRelease(
incoming = crel.payload,
meta = xact_meta.coh,
src = crel.header.src)
val next_coh_on_gnt = co.managerMetadataOnGrant(
outgoing = cgnt.payload,
meta = xact_meta.coh,
dst = cgnt.header.dst)
val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)),
val outer_write = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = macq_data_cnt,
data = xact_data(macq_data_cnt)),
{ case TLId => outerId })
val outer_read = Bundle(UncachedReadBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block),
{ case TLId => outerId })
val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId })
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr)
io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh)
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh),
xact_client_xact_id,
UInt(trackerId),
xact_data(cgnt_data_cnt))
io.inner.grant.bits.payload := xact.makeGrant(
manager_xact_id = UInt(trackerId),
meta = xact_meta.coh,
addr_beat = cgnt_data_cnt,
data = xact_data(cgnt_data_cnt))
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
io.data.read.valid := Bool(false)
io.data.read.bits.id := UInt(trackerId)
io.data.read.bits.way_en := xact_way_en
io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt)
io.data.read.bits.addr_block := xact_addr_block
io.data.read.bits.addr_beat := read_data_cnt
io.data.write.valid := Bool(false)
io.data.write.bits.id := UInt(trackerId)
io.data.write.bits.way_en := xact_way_en
io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt)
io.data.write.bits.addr_block := xact_addr_block
io.data.write.bits.addr_beat := write_data_cnt
io.data.write.bits.wmask := SInt(-1)
io.data.write.bits.data := xact_data(local_data_write_cnt)
io.data.write.bits.data := xact_data(write_data_cnt)
io.meta.read.valid := Bool(false)
io.meta.read.bits.id := UInt(trackerId)
io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB)
io.meta.read.bits.tag := xact_addr >> UInt(idxBits)
io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB)
io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits)
io.meta.write.valid := Bool(false)
io.meta.write.bits.id := UInt(trackerId)
io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB)
io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB)
io.meta.write.bits.way_en := xact_way_en
io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits)
io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits)
io.meta.write.bits.data.coh := next_coh_on_gnt
io.wb.req.valid := Bool(false)
io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB))
io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB))
io.wb.req.bits.coh := xact_meta.coh
io.wb.req.bits.way_en := xact_way_en
io.wb.req.bits.id := UInt(trackerId)
@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
when(collect_cacq_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_data(cacq_data_cnt) := c_acq.payload.data
xact_data(cacq.payload.addr_beat) := cacq.payload.data
cacq_data_valid(cacq.payload.addr_beat) := Bool(true)
}
when(cacq_data_done) { collect_cacq_data := Bool(false) }
}
@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
is(s_idle) {
io.inner.acquire.ready := Bool(true)
when( io.inner.acquire.valid ) {
xact_uncached := c_acq.payload.uncached
xact_a_type := c_acq.payload.a_type
xact_addr := c_acq.payload.addr
xact_client_xact_id := c_acq.payload.client_xact_id
xact_data(UInt(0)) := c_acq.payload.data
xact_subblock := c_acq.payload.subblock
xact_src := c_acq.header.src
collect_cacq_data := co.messageHasData(c_acq.payload)
xact_uncached := cacq.payload.uncached
xact_a_type := cacq.payload.a_type
xact_addr_block := cacq.payload.addr_block
xact_addr_beat := cacq.payload.addr_beat
xact_client_xact_id := cacq.payload.client_xact_id
xact_data(UInt(0)) := cacq.payload.data
xact_subblock := cacq.payload.subblock
xact_src := cacq.header.src
collect_cacq_data := cacq.payload.hasMultibeatData()
state := s_meta_read
}
}
@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
xact_way_en := io.meta.resp.bits.way_en
val coh = io.meta.resp.bits.meta.coh
val _tag_match = io.meta.resp.bits.tag_match
val _needs_writeback = !_tag_match && co.needsWriteback(coh)
val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit
val _needs_probes = _tag_match && co.requiresProbes(xact, coh)
val _is_hit = _tag_match && co.isHit(xact, coh)
val full_block = !xact.uncached ||
xact.is(Acquire.uncachedReadBlock) ||
xact.is(Acquire.uncachedWriteBlock)
read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat)
write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat)
write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat)
resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat)
cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat)
cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat)
when(_needs_probes) {
pending_probes := mask_incoherent(nCoherentClients-1,0)
release_count := co.dir.count(mask_incoherent)
@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
pending_probes := co.dir.pop(pending_probes, curr_p_id)
}
// Handle releases, which may have data being written back
//TODO: make sure cacq data is actually present before accpeting
// release data to merge!
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
xact_meta.coh := next_coh_on_rel
// Handle released dirty data
when(co.messageHasData(c_rel.payload)) {
when(crel.payload.hasData()) {
crel_had_data := Bool(true)
//TODO make sure cacq data is actually present before merging
xact_data(crel_data_cnt) := mergeData(xact,
xact_data(crel_data_cnt),
c_rel.payload.data)
mergeData(xact_data, crel.payload)
}
// We don't decrement release_count until we've received all the data beats.
when(!co.messageHasData(c_rel.payload) || crel_data_done) {
when(!crel.payload.hasMultibeatData() || crel_data_done) {
release_count := release_count - UInt(1)
}
}
@ -847,51 +999,55 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
io.outer.grant.ready := Bool(true)
when(io.outer.grant.valid) {
//TODO make sure cacq data is actually present before merging
xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt),
io.outer.grant.bits.payload.data)
//TODO: set pending client state in xact_meta.coh
when(outer_data_resp_done) {
state := Mux(co.messageHasData(io.outer.grant.bits.payload),
s_data_write, s_data_read)
mergeData(xact_data, mgnt.payload)
when(mgnt_data_done) {
state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read)
}
}
}
is(s_data_read) {
io.data.read.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt))
io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt)
when(io.data.resp.valid) {
xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt),
io.data.resp.bits.data)
mergeData(xact_data, io.data.resp.bits)
resp_data_cnt := resp_data_cnt + UInt(1)
}
when(io.data.read.ready) {
read_data_cnt := read_data_cnt + UInt(1)
when(read_data_cnt === read_data_max) { state := s_data_resp }
}
when(local_data_read_done) { state := s_data_resp }
}
is(s_data_resp) {
when(io.data.resp.valid) {
xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt),
io.data.resp.bits.data)
mergeData(xact_data, io.data.resp.bits)
resp_data_cnt := resp_data_cnt + UInt(1)
}
when(local_data_resp_done) {
state := Mux(co.messageHasData(xact), s_data_write, s_meta_write)
when(resp_data_cnt === resp_data_max) {
state := Mux(xact.hasData(), s_data_write, s_meta_write)
}
}
is(s_data_write) {
io.data.write.valid := Bool(true)
when(local_data_write_done) {
when(io.data.write.ready) {
write_data_cnt := write_data_cnt + UInt(1)
when(write_data_cnt === write_data_max) {
state := s_meta_write
}
}
}
is(s_meta_write) {
io.meta.write.valid := Bool(true)
when(io.meta.write.ready) { state := s_grant }
}
is(s_grant) {
io.inner.grant.valid := Bool(true)
when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) {
state := Mux(co.requiresAckForGrant(c_gnt.payload),
s_busy, s_idle)
when(io.inner.grant.ready) {
cgnt_data_cnt := cgnt_data_cnt + UInt(1)
when(cgnt_data_cnt === cgnt_data_max) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_busy) {
}
is(s_ack) {
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}

1109
uncore/src/main/scala/coherence.scala
View File

File diff suppressed because it is too large Load Diff

6
uncore/src/main/scala/consts.scala
View File

@ -16,7 +16,6 @@ trait MemoryOpConstants {
val MT_BU = Bits("b100")
val MT_HU = Bits("b101")
val MT_WU = Bits("b110")
val MT_CB = Bits("b111") // cache block
val NUM_XA_OPS = 9
val M_SZ = 5
@ -37,8 +36,9 @@ trait MemoryOpConstants {
val M_XA_MAX = Bits("b01101");
val M_XA_MINU = Bits("b01110");
val M_XA_MAXU = Bits("b01111");
val M_INV = Bits("b10000"); // write back and invalidate line
val M_CLN = Bits("b10001"); // write back line
val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions
val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions
val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions
def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP
def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW

26
uncore/src/main/scala/htif.scala
View File

@ -136,7 +136,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
mem_acked := Bool(true)
mem_gxid := io.mem.grant.bits.payload.manager_xact_id
mem_gsrc := io.mem.grant.bits.header.src
mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload)
mem_needs_ack := io.mem.grant.bits.payload.requiresAck()
}
io.mem.grant.ready := Bool(true)
@ -184,22 +184,26 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx)
}
var mem_req_data: UInt = null
for (i <- 0 until dataBits/short_request_bits) {
val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits)))
val n = dataBits/short_request_bits
val mem_req_data = (0 until n).map { i =>
val ui = UInt(i, log2Up(n))
when (state === state_mem_rresp && io.mem.grant.valid) {
packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits)
}
mem_req_data = Cat(packet_ram(idx), mem_req_data)
packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) :=
io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits)
}
packet_ram(Cat(cnt, ui))
}.reverse.reduce(_##_)
val init_addr = addr.toUInt >> UInt(offsetBits-3)
io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq
io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem,
UncachedWrite(init_addr, mem_req_data),
UncachedRead(init_addr))
UncachedWriteBlock(
addr_block = init_addr,
addr_beat = cnt,
client_xact_id = UInt(0),
data = mem_req_data),
UncachedReadBlock(addr_block = init_addr))
io.mem.acquire.bits.payload.data := mem_req_data
io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id
io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module
io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack
io.mem.finish.bits.payload.manager_xact_id := mem_gxid
io.mem.finish.bits.header.dst := mem_gsrc

14
uncore/src/main/scala/memserdes.scala
View File

@ -228,7 +228,9 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
mem_cmd_q.io.enq.valid := Bool(false)
mem_data_q.io.enq.valid := Bool(false)
val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload)
//TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are
//acceptable Acquire types
val acq_has_data = io.uncached.acquire.bits.payload.hasData()
val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats)
val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats)
val active_out = Reg(init=Bool(false))
@ -250,7 +252,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := io.uncached.acquire.bits.payload.client_xact_id
addr_out := io.uncached.acquire.bits.payload.addr
addr_out := io.uncached.acquire.bits.payload.addr_block
has_data := acq_has_data
tl_done_out := tl_wrap_out
mif_done_out := Bool(false)
@ -323,8 +325,12 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
when(tl_wrap_in) { active_in := Bool(false) }
}
io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0),
tl_buf_in(tl_cnt_in))
io.uncached.grant.bits.payload := Grant(uncached = Bool(true),
g_type = Grant.uncachedReadBlock,
client_xact_id = tag_in,
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = tl_buf_in(tl_cnt_in))
}
class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module

399
uncore/src/main/scala/tilelink.scala
View File

@ -16,20 +16,22 @@ case object TLDataBits extends Field[Int]
case object TLDataBeats extends Field[Int]
abstract trait TileLinkParameters extends UsesParameters {
val tlAddrBits = params(TLAddrBits)
val tlBlockAddrBits = params(TLAddrBits)
val tlClientXactIdBits = params(TLClientXactIdBits)
val tlManagerXactIdBits = params(TLManagerXactIdBits)
val tlDataBits = params(TLDataBits)
val tlDataBeats = params(TLDataBeats)
val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits
val tlSubblockAddrBits = log2Up(tlWriteMaskBits)
val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS)
val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8
val tlBeatAddrBits = log2Up(tlDataBeats)
val tlByteAddrBits = log2Up(tlWriteMaskBits)
val tlAtomicOpcodeBits = M_SZ
val tlUncachedOperandSizeBits = MT_SZ
val tlSubblockUnionBits = max(tlWriteMaskBits,
(tlSubblockAddrBits +
(tlByteAddrBits +
tlUncachedOperandSizeBits +
tlAtomicOpcodeBits)) + 1
val co = params(TLCoherence)
val networkPreservesPointToPointOrdering = false //TODO: check physical network type
}
abstract class TLBundle extends Bundle with TileLinkParameters
@ -45,8 +47,12 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now
// Common signals that are used in multiple channels.
// These traits are useful for type parameterization.
//
trait HasPhysicalAddress extends TLBundle {
val addr = UInt(width = tlAddrBits)
trait HasCacheBlockAddress extends TLBundle {
val addr_block = UInt(width = tlBlockAddrBits)
}
trait HasTileLinkBeatId extends TLBundle {
val addr_beat = UInt(width = tlBeatAddrBits)
}
trait HasClientTransactionId extends TLBundle {
@ -57,55 +63,109 @@ trait HasManagerTransactionId extends TLBundle {
val manager_xact_id = Bits(width = tlManagerXactIdBits)
}
trait HasTileLinkData extends TLBundle {
abstract trait HasTileLinkData extends HasTileLinkBeatId {
val data = UInt(width = tlDataBits)
def hasData(dummy: Int = 0): Bool
def hasMultibeatData(dummy: Int = 0): Bool
}
// Actual TileLink channel bundle definitions
class Acquire extends ClientToManagerChannel
with HasPhysicalAddress
with HasCacheBlockAddress
with HasClientTransactionId
with HasTileLinkData {
// Actual bundle fields
val uncached = Bool()
val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth))
val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth))
val subblock = Bits(width = tlSubblockUnionBits)
val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits
val opSzOff = tlUncachedOperandSizeBits + sbAddrOff
// Utility funcs for accessing uncached/subblock union
val opSizeOff = tlByteAddrBits + 1
val opCodeOff = tlUncachedOperandSizeBits + opSizeOff
val opMSB = tlAtomicOpcodeBits + opCodeOff
def allocate(dummy: Int = 0) = subblock(0)
def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1)
def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1)
def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1)
def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1)
def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff)
def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff)
def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1)
def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0))
// Other helper funcs
def is(t: UInt) = a_type === t
def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type)
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached &&
Acquire.typesWithMultibeatData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type)
def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe =
Probe(co.getProbeType(this, meta), this.addr_block)
def makeGrant(
manager_xact_id: UInt,
meta: ManagerMetadata = co.managerMetadataOnFlush,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
Grant(
uncached = this.uncached,
g_type = co.getGrantType(this, meta),
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id,
addr_beat = addr_beat,
data = data
)
}
}
object Acquire {
val nUncachedAcquireTypes = 3
//TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes)
val nBuiltinAcquireTypes = 5
//TODO: Use Enum
def uncachedRead = UInt(0)
def uncachedWrite = UInt(1)
def uncachedAtomic = UInt(2)
def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type)
def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite
def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite
def uncachedReadBlock = UInt(1)
def uncachedWrite = UInt(2)
def uncachedWriteBlock = UInt(3)
def uncachedAtomic = UInt(4)
def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic)
def typesWithMultibeatData = Vec(uncachedWriteBlock)
def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock
def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock
def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = {
def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt
// Most generic constructor
def apply(
uncached: Bool,
a_type: Bits,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0),
subblock: UInt = UInt(0)): Acquire = {
val acq = new Acquire
acq.uncached := uncached
acq.a_type := a_type
acq.addr := addr
acq.client_xact_id := client_xact_id
acq.addr_block := addr_block
acq.addr_beat := addr_beat
acq.data := data
acq.subblock := subblock
acq
}
def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0))
}
def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = {
apply(a_type, addr, client_xact_id, UInt(0))
// For cached types
def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = {
apply(
uncached = Bool(false),
a_type = a_type,
client_xact_id = client_xact_id,
addr_block = addr_block)
}
// Copy constructor
def apply(a: Acquire): Acquire = {
val acq = new Acquire
acq := a
@ -113,83 +173,189 @@ object Acquire {
}
}
// Asks for a single TileLink beat of data
object UncachedRead {
def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = {
val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id)
acq.uncached := Bool(true)
acq.subblock := Cat(subblock_addr, operand_sz, alloc)
acq
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
uncached = Bool(true),
a_type = Acquire.uncachedRead,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
subblock = alloc)
}
def apply(addr: UInt, client_xact_id: UInt): Acquire = {
apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true))
}
def apply(addr: UInt): Acquire = {
apply(addr, UInt(0))
}
// Asks for an entire cache block of data
object UncachedReadBlock {
def apply(
client_xact_id: UInt = UInt(0),
addr_block: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
uncached = Bool(true),
a_type = Acquire.uncachedReadBlock,
client_xact_id = client_xact_id,
addr_block = addr_block,
subblock = alloc.toUInt)
}
}
object UncachedWrite {
def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = {
val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data)
acq.uncached := Bool(true)
acq.subblock := Cat(write_mask, alloc)
acq
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
data: UInt,
write_mask: UInt = Acquire.fullWriteMask,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
uncached = Bool(true),
a_type = Acquire.uncachedWrite,
addr_block = addr_block,
addr_beat = addr_beat,
client_xact_id = client_xact_id,
data = data,
subblock = Cat(write_mask, alloc))
}
def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
apply(addr, client_xact_id, SInt(-1), Bool(true), data)
}
def apply(addr: UInt, data: UInt): Acquire = {
apply(addr, UInt(0), data)
}
// For full block of data
object UncachedWriteBlock {
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
data: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
uncached = Bool(true),
a_type = Acquire.uncachedWriteBlock,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
subblock = Cat(Acquire.fullWriteMask, alloc))
}
}
object UncachedAtomic {
def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt,
subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = {
val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data)
acq.uncached := Bool(true)
acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true))
acq
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
addr_byte: UInt,
atomic_opcode: UInt,
operand_size: UInt,
data: UInt): Acquire = {
Acquire(
uncached = Bool(true),
a_type = Acquire.uncachedAtomic,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true)))
}
}
class Probe extends ManagerToClientChannel
with HasPhysicalAddress {
with HasCacheBlockAddress {
val p_type = UInt(width = co.probeTypeWidth)
def is(t: UInt) = p_type === t
def makeRelease(
client_xact_id: UInt,
meta: ClientMetadata = co.clientMetadataOnFlush,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(false),
r_type = co.getReleaseType(this, meta),
client_xact_id = client_xact_id,
addr_block = this.addr_block,
addr_beat = addr_beat,
data = data)
}
}
object Probe {
def apply(p_type: UInt, addr: UInt) = {
val co = new Probe().co
def apply(p_type: UInt, addr_block: UInt) = {
val prb = new Probe
prb.p_type := p_type
prb.addr := addr
prb.addr_block := addr_block
prb
}
def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = {
apply(co.getProbeType(M_FLUSH, meta), addr_block)
}
}
class Release extends ClientToManagerChannel
with HasPhysicalAddress
with HasCacheBlockAddress
with HasClientTransactionId
with HasTileLinkData {
val r_type = UInt(width = co.releaseTypeWidth)
val voluntary = Bool()
// Helper funcs
def is(t: UInt) = r_type === t
def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type)
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type)
def isVoluntary(dummy: Int = 0) = voluntary
def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering)
def makeGrant(
manager_xact_id: UInt,
meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = {
Grant(
g_type = Grant.voluntaryAck,
uncached = Bool(true), // Grant.voluntaryAck is built-in type
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id
)
}
}
object Release {
def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = {
val co = new Release().co
def apply(
voluntary: Bool,
r_type: UInt,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
val rel = new Release
rel.r_type := r_type
rel.addr := addr
rel.client_xact_id := client_xact_id
rel.addr_block := addr_block
rel.addr_beat := addr_beat
rel.data := data
rel.voluntary := voluntary
rel
}
def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = {
apply(r_type, addr, client_xact_id, UInt(0))
}
def apply(r_type: UInt, addr: UInt): Release = {
apply(r_type, addr, UInt(0), UInt(0))
def makeVoluntaryWriteback(
meta: ClientMetadata,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(true),
r_type = co.getReleaseType(M_FLUSH, meta),
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data)
}
}
@ -198,29 +364,62 @@ class Grant extends ManagerToClientChannel
with HasClientTransactionId
with HasManagerTransactionId {
val uncached = Bool()
val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth))
val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth))
// Helper funcs
def is(t: UInt) = g_type === t
def hasData(dummy: Int = 0): Bool = Mux(uncached,
Grant.typesWithData.contains(g_type),
co.grantTypesWithData.contains(g_type))
def hasMultibeatData(dummy: Int = 0): Bool =
Bool(tlDataBeats > 1) && Mux(uncached,
Grant.typesWithMultibeatData.contains(g_type),
co.grantTypesWithData.contains(g_type))
def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck)
def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary()
def makeFinish(dummy: Int = 0): Finish = {
val f = new Finish
f.manager_xact_id := this.manager_xact_id
f
}
}
object Grant {
val nUncachedGrantTypes = 3
//TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes)
def uncachedRead = UInt(0)
def uncachedWrite = UInt(1)
def uncachedAtomic = UInt(2)
def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type)
val nBuiltinGrantTypes = 5
//TODO Use Enum
def voluntaryAck = UInt(0)
def uncachedRead = UInt(1)
def uncachedReadBlock = UInt(2)
def uncachedWrite = UInt(3)
def uncachedAtomic = UInt(4)
def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic)
def typesWithMultibeatData= Vec(uncachedReadBlock)
def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = {
def apply(
uncached: Bool,
g_type: UInt,
client_xact_id: UInt,
manager_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
val gnt = new Grant
gnt.uncached := uncached
gnt.g_type := g_type
gnt.client_xact_id := client_xact_id
gnt.manager_xact_id := manager_xact_id
gnt.addr_beat := addr_beat
gnt.data := data
gnt
}
def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = {
apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0))
def getGrantTypeForUncached(a: Acquire): UInt = {
MuxLookup(a.a_type, Grant.uncachedRead, Array(
Acquire.uncachedRead -> Grant.uncachedRead,
Acquire.uncachedReadBlock -> Grant.uncachedReadBlock,
Acquire.uncachedWrite -> Grant.uncachedWrite,
Acquire.uncachedWriteBlock -> Grant.uncachedWrite,
Acquire.uncachedAtomic -> Grant.uncachedAtomic
))
}
}
@ -260,10 +459,17 @@ object TileLinkIOWrapper {
}
}
// Utility functions for constructing TileLinkIO arbiters
abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule {
abstract trait HasArbiterTypes {
val arbN: Int
type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId
type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId
type ClientSourcedWithIdAndData = ClientToManagerChannel with
HasClientTransactionId with
HasTileLinkData
}
// Utility functions for constructing TileLinkIO arbiters
abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule
with HasArbiterTypes {
// These are filled in depending on whether the arbiter mucks with the
// client ids and then needs to revert them on the way back
@ -271,10 +477,10 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule {
def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits
def arbIdx(in: ManagerSourcedWithId): UInt
def hookupClientSource[M <: ClientSourcedWithId]
def hookupClientSource[M <: ClientSourcedWithIdAndData]
(ins: Seq[DecoupledIO[LogicalNetworkIO[M]]],
out: DecoupledIO[LogicalNetworkIO[M]]) {
def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload)
def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData()
val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _)))
out <> arb.io.out
ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => {
@ -336,35 +542,26 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) {
}
// Appends the port index of the arbiter to the client_xact_id
abstract trait AppendsArbiterId {
val arbN: Int
def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) =
abstract trait AppendsArbiterId extends HasArbiterTypes {
def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) =
Cat(in.client_xact_id, UInt(id, log2Up(arbN)))
def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) =
def managerSourcedClientXactId(in: ManagerSourcedWithId) =
in.client_xact_id >> UInt(log2Up(arbN))
def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) =
in.client_xact_id(log2Up(arbN)-1,0).toUInt
def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt
}
// Uses the client_xact_id as is (assumes it has been set to port index)
abstract trait PassesId {
def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) =
in.client_xact_id
def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) =
in.client_xact_id
def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) =
in.client_xact_id
abstract trait PassesId extends HasArbiterTypes {
def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id
def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id
def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id
}
// Overwrites some default client_xact_id with the port idx
abstract trait UsesNewId {
val arbN: Int
def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) =
UInt(id, log2Up(arbN))
def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) =
UInt(0)
def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) =
in.client_xact_id
abstract trait UsesNewId extends HasArbiterTypes {
def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN))
def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0)
def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id
}
// Mix-in id generation traits to make concrete arbiter classes

197
uncore/src/main/scala/uncore.scala
View File

@ -67,7 +67,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends
val sdq_val = Reg(init=Bits(0, sdqDepth))
val sdq_alloc_id = PriorityEncoder(~sdq_val)
val sdq_rdy = !sdq_val.andR
val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload)
val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
@ -87,8 +87,8 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends
// Queue to store impending Voluntary Release data
val release = io.inner.release
val voluntary = co.isVoluntary(release.bits.payload)
val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload)
val voluntary = release.bits.payload.isVoluntary()
val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width
val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1
when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
@ -117,9 +117,10 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends
probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe }
// Wire grant reply to initiating client
def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload)
def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData()
val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _)))
io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat
io.inner.grant <> grant_arb.io.out
grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant }
@ -130,7 +131,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
co.messageHasData(io.outer.acquire.bits.payload) &&
io.outer.acquire.bits.payload.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
@ -156,32 +157,39 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module {
val has_release_match = Bool(OUTPUT)
}
val c_acq = io.inner.acquire.bits
val c_rel = io.inner.release.bits
val c_gnt = io.inner.grant.bits
val c_ack = io.inner.finish.bits
val m_gnt = io.outer.grant.bits
val cacq = io.inner.acquire.bits
val crel = io.inner.release.bits
val cgnt = io.inner.grant.bits
val cfin = io.inner.finish.bits
val macq = io.outer.acquire.bits
val mgnt = io.outer.grant.bits
}
class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4)
val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.release.bits.header.src.clone)
val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
val xact_addr = Reg(io.inner.release.bits.payload.addr.clone)
val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone)
val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
val xact = Release(
voluntary = Bool(true),
r_type = xact_r_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block)
val collect_inner_data = Reg(init=Bool(false))
// TODO: assert that all releases have full blocks of data
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats)
val (outer_data_cnt, outer_data_done) =
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats)
io.has_acquire_conflict := Bool(false)
io.has_release_match := co.isVoluntary(c_rel.payload)
io.has_release_match := crel.payload.isVoluntary()
io.outer.grant.ready := Bool(false)
io.outer.acquire.valid := Bool(false)
@ -193,21 +201,19 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := Grant(Bool(false),
co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush),
xact_client_xact_id,
UInt(trackerId))
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId))
io.outer.acquire.bits.payload := Bundle(UncachedWrite(
xact_addr,
UInt(trackerId),
xact_data(outer_data_cnt)),
io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
when(collect_inner_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
xact_data(inner_data_cnt) := c_rel.payload.data
xact_data(inner_data_cnt) := crel.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
}
@ -216,72 +222,107 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := c_rel.header.src
xact_r_type := c_rel.payload.r_type
xact_addr := c_rel.payload.addr
xact_client_xact_id := c_rel.payload.client_xact_id
xact_data(UInt(0)) := c_rel.payload.data
collect_inner_data := co.messageHasData(c_rel.payload)
state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack)
xact_src := crel.header.src
xact_r_type := crel.payload.r_type
xact_addr_block := crel.payload.addr_block
xact_client_xact_id := crel.payload.client_xact_id
xact_data(UInt(0)) := crel.payload.data
collect_inner_data := crel.payload.hasMultibeatData()
state := Mux(crel.payload.hasData(), s_outer,
Mux(crel.payload.requiresAck(), s_ack, s_idle))
}
}
is(s_outer) {
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
when(outer_data_done) { state := s_ack }
when(outer_data_done) {
state := Mux(xact.requiresAck(), s_grant, s_idle)
}
}
is(s_grant) {
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) {
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) { state := s_idle }
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
}
class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6)
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone)
val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone)
val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock)
val xact = Acquire(
uncached = xact_uncached,
a_type = xact_a_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block,
addr_beat = xact_addr_beat,
data = UInt(0),
subblock = xact_subblock)
val collect_inner_data = Reg(init=Bool(false))
//TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats)
Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats)
val (outer_data_cnt, outer_data_done) =
Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats)
val (cgnt_data_cnt, cgnt_data_done) =
Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats)
val release_count = Reg(init=UInt(0, width = log2Up(nClients)))
val probe_flags = Reg(init=Bits(0, width = nClients))
val curr_p_id = PriorityEncoder(probe_flags)
val pending_outer_write = co.messageHasData(xact)
val pending_outer_write = xact.hasData()
val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush)
val probe_initial_flags = Bits(width = nClients)
probe_initial_flags := Bits(0)
// issue self-probes for uncached read xacts to facilitate I$ coherence
val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload)
val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0)))
val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe()
val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0)))
probe_initial_flags := ~(io.tile_incoherent | myflag)
io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) &&
io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) &&
(state != s_idle) &&
!collect_inner_data
io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) &&
!co.isVoluntary(c_rel.payload) &&
io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) &&
!crel.payload.isVoluntary() &&
(state != s_idle)
val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)),
val outer_write_acq = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data),
val outer_write_rel = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = crel.payload.addr_beat,
data = crel.payload.data),
{ case TLId => outerId })
val outer_read = Bundle(UncachedReadBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block),
{ case TLId => outerId })
val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId })
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
@ -290,16 +331,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr)
io.inner.probe.bits.payload := xact.makeProbe()
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := Grant(xact_uncached,
co.getGrantType(xact, co.managerMetadataOnFlush),
xact_client_xact_id,
UInt(trackerId),
UInt(0)) // Data bypassed in parent
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
@ -307,7 +344,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
when(collect_inner_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_data(inner_data_cnt) := c_acq.payload.data
xact_data(inner_data_cnt) := cacq.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
}
@ -315,17 +352,18 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
switch (state) {
is(s_idle) {
io.inner.acquire.ready := Bool(true)
val needs_outer_write = co.messageHasData(c_acq.payload)
val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush)
when( io.inner.acquire.valid ) {
xact_uncached := c_acq.payload.uncached
xact_a_type := c_acq.payload.a_type
xact_addr := c_acq.payload.addr
xact_client_xact_id := c_acq.payload.client_xact_id
xact_data(UInt(0)) := c_acq.payload.data
xact_subblock := c_acq.payload.subblock
xact_src := c_acq.header.src
collect_inner_data := co.messageHasData(c_acq.payload)
val needs_outer_write = cacq.payload.hasData()
val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush)
when(io.inner.acquire.valid) {
xact_uncached := cacq.payload.uncached
xact_a_type := cacq.payload.a_type
xact_addr_block := cacq.payload.addr_block
xact_addr_beat := cacq.payload.addr_beat
xact_client_xact_id := cacq.payload.client_xact_id
xact_data(UInt(0)) := cacq.payload.data
xact_subblock := cacq.payload.subblock
xact_src := cacq.header.src
collect_inner_data := cacq.payload.hasMultibeatData()
probe_flags := probe_initial_flags
release_count := PopCount(probe_initial_flags)
state := Mux(probe_initial_flags.orR, s_probe,
@ -341,9 +379,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
}
// Handle releases, which may have data to be written back
io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready
io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready
when(io.inner.release.valid) {
when(co.messageHasData(c_rel.payload)) {
when(crel.payload.hasData()) {
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_write_rel
when(io.outer.acquire.ready) {
@ -364,14 +402,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
}
}
}
is(s_mem_read) {
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_read
when(io.outer.acquire.ready) {
state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle)
when(io.outer.acquire.ready) { state := s_mem_resp }
}
}
is(s_mem_write) {
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
io.outer.acquire.bits.payload := outer_write_acq
@ -379,17 +415,22 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
state := Mux(pending_outer_read, s_mem_read, s_make_grant)
}
}
is(s_make_grant) {
is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle)
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_busy) { // Nothing left to do but wait for transaction to complete
when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) {
is(s_mem_resp) { // Wait to forward grants from outer memory
when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) {
io.inner.grant.valid := Bool(true)
}
when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) {
when(cgnt_data_done) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) {
state := s_idle
}
}

6
uncore/src/main/scala/util.scala
View File

@ -34,7 +34,7 @@ object ZCounter {
}
}
class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module {
class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module {
val io = new Bundle {
val in = Decoupled(gen.clone).flip
val out = Decoupled(gen.clone)
@ -65,12 +65,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I
io.out.valid := active || io.in.valid
io.out.bits := io.in.bits
when(!active && io.in.valid) {
when(doSer(io.in.bits.payload)) {
when(io.in.bits.payload.hasData()) {
cnt := Mux(io.out.ready, UInt(1), UInt(0))
rbits := io.in.bits
active := Bool(true)
}
io.done := !doSer(io.in.bits.payload)
io.done := !io.in.bits.payload.hasData()
}
when(active) {
io.out.bits := rbits