riscv/rocket-chip
1
0
Fork 0
Code Releases Activity

New metadata-based coherence API

Browse Source
This commit is contained in:
Henry Cook 2015-02-28 17:02:13 -08:00
parent 0a8722e881
commit 1bed6ea498
9 changed files with 2117 additions and 1851 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

380
uncore/src/main/scala/broadcast.scala Normal file
View File

@ -0,0 +1,380 @@
// See LICENSE for license details.
package uncore
import Chisel._
case object L2StoreDataQueueDepth extends Field[Int]
trait BroadcastHubParameters extends CoherenceAgentParameters {
val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
}
class DataQueueLocation extends Bundle with BroadcastHubParameters {
val idx = UInt(width = dqIdxBits)
val loc = UInt(width = log2Ceil(nDataQueueLocations))
}
object DataQueueLocation {
def apply(idx: UInt, loc: UInt) = {
val d = new DataQueueLocation
d.idx := idx
d.loc := loc
d
}
}
class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent
with BroadcastHubParameters {
val internalDataBits = new DataQueueLocation().getWidth
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
// Create SHRs for outstanding transactions
val trackerList = (0 until nReleaseTransactors).map(id =>
Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++
(nReleaseTransactors until nTransactors).map(id =>
Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits}))
// Propagate incoherence flags
trackerList.map(_.io.incoherent := io.incoherent.toBits)
// Queue to store impending Put data
val acquire = io.inner.acquire
val sdq_val = Reg(init=Bits(0, sdqDepth))
val sdq_alloc_id = PriorityEncoder(~sdq_val)
val sdq_rdy = !sdq_val.andR
val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
// Handle acquire transaction initiation
val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
val block_acquires = any_acquire_conflict
val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
alloc_arb.io.in(i).valid := t.acquire.ready
t.acquire.bits := acquire.bits
t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
t.acquire.valid := alloc_arb.io.in(i).ready
}
acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
// Queue to store impending Voluntary Release data
val release = io.inner.release
val voluntary = release.bits.payload.isVoluntary()
val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1
when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
// Handle releases, which might be voluntary and might have data
val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b}
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
t.release.bits := release.bits
t.release.bits.payload.data := (if (i < nReleaseTransactors)
DataQueueLocation(rel_data_cnt, inVolWBQueue)
else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits
t.release.valid := release.valid && (release_idx === UInt(i))
}
release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx)
// Wire finished transaction acks
val ack = io.inner.finish
trackerList.map(_.io.inner.finish.valid := ack.valid)
trackerList.map(_.io.inner.finish.bits := ack.bits)
ack.ready := Bool(true)
// Wire probe requests and grant reply to clients, finish acks from clients
// Note that we bypass the Grant data subbundles
io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
// Create an arbiter for the one memory port
val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
{ case TLId => params(OuterTLId)
case TLDataBits => internalDataBits })
outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer }
// Get the pending data out of the store data queue
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.payload.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
io.outer <> outer_arb.io.out
// Update SDQ valid bits
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
}
class BroadcastXactTracker extends XactTracker {
val io = new ManagerXactTrackerIO
}
class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker {
val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.release.bits.header.src.clone)
val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 }))
val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) }
val coh = ManagerMetadata.onReset
val collect_irel_data = Reg(init=Bool(false))
val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats))
val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
io.has_acquire_conflict := Bool(false)
io.has_release_match := io.irel().isVoluntary()
io.has_acquire_match := Bool(false)
io.outer.acquire.valid := Bool(false)
io.outer.grant.ready := Bool(false)
io.outer.finish.valid := Bool(false)
io.inner.acquire.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.grant.valid := Bool(false)
io.inner.finish.ready := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId))
//TODO: Use io.outer.release instead?
io.outer.acquire.bits.payload := Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = oacq_data_cnt,
data = data_buffer(oacq_data_cnt)))(outerTLParams)
when(collect_irel_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
data_buffer(io.irel().addr_beat) := io.irel().data
irel_data_valid(io.irel().addr_beat) := Bool(true)
}
when(irel_data_done) { collect_irel_data := Bool(false) }
}
switch (state) {
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := io.inner.release.bits.header.src
xact := io.irel()
data_buffer(UInt(0)) := io.irel().data
collect_irel_data := io.irel().hasMultibeatData()
irel_data_valid := io.irel().hasData() << io.irel().addr_beat
state := Mux(io.irel().hasData(), s_outer,
Mux(io.irel().requiresAck(), s_ack, s_idle))
}
}
is(s_outer) {
io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt)
when(oacq_data_done) {
state := Mux(xact.requiresAck(), s_grant, s_idle)
}
}
is(s_grant) { // Forward the Grant.voluntaryAck
io.outer.grant.ready := io.inner.grant.ready
io.inner.grant.valid := io.outer.grant.valid
when(io.inner.grant.fire()) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_ack) {
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
}
class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker {
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 }))
val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) }
val coh = ManagerMetadata.onReset
assert(!(state != s_idle && xact.isBuiltInType() &&
Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)),
"Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO
val release_count = Reg(init=UInt(0, width = log2Up(nClients)))
val probe_flags = Reg(init=Bits(0, width = nClients))
val curr_p_id = PriorityEncoder(probe_flags)
val probe_initial_flags = Bits(width = nClients)
probe_initial_flags := Bits(0)
// issue self-probes for uncached read xacts to facilitate I$ coherence
val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe()
val myflag = Mux(probe_self, Bits(0),
UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0)))
probe_initial_flags := ~(io.incoherent.toBits | myflag)
val collect_iacq_data = Reg(init=Bool(false))
val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats))
val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire)
val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant)
val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
val pending_ognt_ack = Reg(init=Bool(false))
val pending_outer_write = xact.hasData()
val pending_outer_read = io.ignt().hasData()
val pending_outer_write_ = io.iacq().hasData()
val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData()
io.has_acquire_conflict := xact.conflicts(io.iacq()) &&
!collect_iacq_data &&
(state != s_idle)
io.has_release_match := xact.conflicts(io.irel()) &&
!io.irel().isVoluntary() &&
(state != s_idle)
io.has_acquire_match := Bool(false)
val outer_write_acq = Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = oacq_data_cnt,
data = data_buffer(oacq_data_cnt)))(outerTLParams)
val outer_write_rel = Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = io.irel().addr_beat,
data = io.irel().data))(outerTLParams)
val outer_read = Bundle(GetBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block))(outerTLParams)
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
io.outer.grant.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := coh.makeProbe(xact)
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.finish.ready := Bool(false)
when(collect_iacq_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
data_buffer(io.iacq().addr_beat) := io.iacq().data
iacq_data_valid(io.iacq().addr_beat) := Bool(true)
}
when(iacq_data_done) { collect_iacq_data := Bool(false) }
}
when(pending_ognt_ack) {
io.outer.grant.ready := Bool(true)
when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) }
//TODO add finish queue if this isnt the last level manager
}
switch (state) {
is(s_idle) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_src := io.inner.acquire.bits.header.src
xact := io.iacq()
data_buffer(UInt(0)) := io.iacq().data
collect_iacq_data := io.iacq().hasMultibeatData()
iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat
probe_flags := probe_initial_flags
release_count := PopCount(probe_initial_flags)
state := Mux(probe_initial_flags.orR, s_probe,
Mux(pending_outer_write_, s_mem_write,
Mux(pending_outer_read_, s_mem_read, s_make_grant)))
}
}
is(s_probe) {
// Generate probes
io.inner.probe.valid := probe_flags.orR
when(io.inner.probe.ready) {
probe_flags := probe_flags & ~(UIntToOH(curr_p_id))
}
// Handle releases, which may have data to be written back
io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready
when(io.inner.release.valid) {
when(io.irel().hasData()) {
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_write_rel
when(io.outer.acquire.ready) {
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
} .otherwise {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
}
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt)
io.outer.acquire.bits.payload := outer_write_acq
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
state := Mux(pending_outer_read, s_mem_read, s_mem_resp)
}
}
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := !pending_ognt_ack
io.outer.acquire.bits.payload := outer_read
when(io.outer.acquire.fire()) { state := s_mem_resp }
}
is(s_mem_resp) { // Wait to forward grants from outer memory
io.outer.grant.ready := io.inner.grant.ready
io.inner.grant.valid := io.outer.grant.valid
when(ignt_data_done) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
}

1232
uncore/src/main/scala/cache.scala
View File

File diff suppressed because it is too large Load Diff

985
uncore/src/main/scala/coherence.scala
View File

File diff suppressed because it is too large Load Diff

43
uncore/src/main/scala/directory.scala Normal file
View File

@ -0,0 +1,43 @@
// See LICENSE for license details.
package uncore
import Chisel._
// This class encapsulates transformations on different directory information
// storage formats
abstract class DirectoryRepresentation(val width: Int) {
def pop(prev: UInt, id: UInt): UInt
def push(prev: UInt, id: UInt): UInt
def flush: UInt
def none(s: UInt): Bool
def one(s: UInt): Bool
def count(s: UInt): UInt
def next(s: UInt): UInt
def full(s: UInt): UInt
}
abstract trait HasDirectoryRepresentation {
val dir: DirectoryRepresentation
}
class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) {
def pop(prev: UInt, id: UInt) = UInt(0)
def push(prev: UInt, id: UInt) = UInt(0)
def flush = UInt(0)
def none(s: UInt) = Bool(false)
def one(s: UInt) = Bool(false)
def count(s: UInt) = UInt(nClients)
def next(s: UInt) = UInt(0)
def full(s: UInt) = SInt(-1, width = nClients).toUInt
}
class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) {
def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id)
def push(prev: UInt, id: UInt) = prev | UIntToOH(id)
def flush = UInt(0, width = width)
def none(s: UInt) = s === UInt(0)
def one(s: UInt) = PopCount(s) === UInt(1)
def count(s: UInt) = PopCount(s)
def next(s: UInt) = PriorityEncoder(s)
def full(s: UInt) = s
}

7
uncore/src/main/scala/htif.scala
View File

@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int]
abstract trait HTIFParameters extends UsesParameters {
val dataBits = params(TLDataBits)
val dataBeats = params(TLDataBeats)
val co = params(TLCoherence)
val w = params(HTIFWidth)
val nSCR = params(HTIFNSCR)
val offsetBits = params(HTIFOffsetBits)
@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
val init_addr = addr.toUInt >> UInt(offsetBits-3)
io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq
io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem,
UncachedWriteBlock(
PutBlock(
addr_block = init_addr,
addr_beat = cnt,
client_xact_id = UInt(0),
data = mem_req_data),
UncachedReadBlock(addr_block = init_addr))
GetBlock(addr_block = init_addr))
io.mem.acquire.bits.payload.data := mem_req_data
io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack
io.mem.finish.bits.payload.manager_xact_id := mem_gxid
@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
for (i <- 0 until scr_rdata.size)
scr_rdata(i) := io.scr.rdata(i)
scr_rdata(0) := UInt(nCores)
scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20)
scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20)
io.scr.wen := Bool(false)
io.scr.wdata := pcr_wdata

312
uncore/src/main/scala/memserdes.scala
View File

@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side
io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w))
}
//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary
//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO
class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
class MemIOTileLinkIOConverter(qDepth: Int) extends Module {
val io = new Bundle {
val uncached = new UncachedTileLinkIO().flip
val tl = new TileLinkIO().flip
val mem = new MemIO
}
val co = params(TLCoherence)
val mifTagBits = params(MIFTagBits)
val mifDataBits = params(MIFDataBits)
val mifDataBeats = params(MIFDataBeats)
@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
val tlDataBeats = params(TLDataBeats)
val dataBits = tlDataBits*tlDataBeats
require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats)
//require(params(TLClientXactIdBits) <= params(MIFTagBits))
require(params(TLClientXactIdBits) <= params(MIFTagBits))
io.tl.acquire.ready := Bool(false)
io.tl.probe.valid := Bool(false)
io.tl.release.ready := Bool(false)
io.tl.finish.ready := Bool(true)
io.mem.resp.ready := Bool(false)
val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2))
io.tl.grant <> gnt_arb.io.out
val acq_has_data = io.tl.acquire.bits.payload.hasData()
val rel_has_data = io.tl.release.bits.payload.hasData()
// Decompose outgoing TL Acquires into MemIO cmd and data
val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth))
val mem_data_q = Module(new Queue(new MemData, qDepth))
io.uncached.acquire.ready := Bool(false)
io.uncached.grant.valid := Bool(false)
io.mem.resp.ready := Bool(false)
mem_cmd_q.io.enq.valid := Bool(false)
mem_data_q.io.enq.valid := Bool(false)
//TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are
//acceptable Acquire types
val acq_has_data = io.uncached.acquire.bits.payload.hasData()
val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats)
val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats)
val active_out = Reg(init=Bool(false))
val cmd_sent_out = Reg(init=Bool(false))
val tl_done_out = Reg(init=Bool(false))
val mif_done_out = Reg(init=Bool(false))
val tag_out = Reg(Bits())
val addr_out = Reg(Bits())
val has_data = Reg(init=Bool(false))
val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) }
val mif_buf_out = Vec.fill(mifDataBeats){ new MemData }
mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits)
val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits)
val tl_prog_out = tl_cnt_out*UInt(tlDataBits)
val data_from_rel = Reg(init=Bool(false))
val (tl_cnt_out, tl_wrap_out) =
Counter((io.tl.acquire.fire() && acq_has_data) ||
(io.tl.release.fire() && rel_has_data), tlDataBeats)
val tl_done_out = Reg(init=Bool(false))
val make_grant_ack = Reg(init=Bool(false))
val grant_for_rel = Grant(
is_builtin_type = Bool(true),
g_type = Grant.voluntaryAckType,
client_xact_id = tag_out,
manager_xact_id = UInt(0))
val grant_for_acq_write = ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = tag_out(0),
a_type = tag_out >> UInt(1),
client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0))
gnt_arb.io.in(1).valid := Bool(false)
gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write)
when(!active_out){
io.uncached.acquire.ready := Bool(true)
when(io.uncached.acquire.valid) {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := io.uncached.acquire.bits.payload.client_xact_id
addr_out := io.uncached.acquire.bits.payload.addr_block
has_data := acq_has_data
tl_done_out := tl_wrap_out
mif_done_out := Bool(false)
tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data
if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) {
val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth))
val mem_data_q = Module(new Queue(new MemData, qDepth))
mem_cmd_q.io.enq.valid := Bool(false)
mem_data_q.io.enq.valid := Bool(false)
val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats)
val mif_done_out = Reg(init=Bool(false))
val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) }
val mif_buf_out = Vec.fill(mifDataBeats){ new MemData }
mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits)
val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits)
val tl_prog_out = tl_cnt_out*UInt(tlDataBits)
when(!active_out){
io.tl.release.ready := Bool(true)
io.tl.acquire.ready := !io.tl.release.valid
when(io.tl.release.valid) {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := io.tl.release.bits.payload.client_xact_id
addr_out := io.tl.release.bits.payload.addr_block
has_data := rel_has_data
data_from_rel := Bool(true)
make_grant_ack := Bool(true)
tl_done_out := tl_wrap_out
tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data
} .elsewhen(io.tl.acquire.valid) {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id,
io.tl.acquire.bits.payload.a_type,
io.tl.acquire.bits.payload.is_builtin_type)
addr_out := io.tl.acquire.bits.payload.addr_block
has_data := acq_has_data
data_from_rel := Bool(false)
make_grant_ack := acq_has_data
tl_done_out := tl_wrap_out
tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data
}
}
}
when(active_out) {
when(!cmd_sent_out) {
mem_cmd_q.io.enq.valid := Bool(true)
}
when(has_data) {
when(!tl_done_out) {
io.uncached.acquire.ready := Bool(true)
when(io.uncached.acquire.valid) {
tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data
when(active_out) {
mem_cmd_q.io.enq.valid := !cmd_sent_out
cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire()
when(has_data) {
when(!tl_done_out) {
io.tl.acquire.ready := Bool(true)
when(io.tl.acquire.valid) {
tl_buf_out(tl_cnt_out) := Mux(data_from_rel,
io.tl.release.bits.payload.data,
io.tl.acquire.bits.payload.data)
}
}
when(!mif_done_out) {
mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out
}
}
when(!mif_done_out) {
mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out
when(tl_wrap_out) { tl_done_out := Bool(true) }
when(mif_wrap_out) { mif_done_out := Bool(true) }
when(tl_done_out && make_grant_ack) {
gnt_arb.io.in(1).valid := Bool(true)
when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) }
}
when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) {
active_out := Bool(false)
}
}
when(mem_cmd_q.io.enq.fire()) {
cmd_sent_out := Bool(true)
mem_cmd_q.io.enq.bits.rw := has_data
mem_cmd_q.io.enq.bits.tag := tag_out
mem_cmd_q.io.enq.bits.addr := addr_out
mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data
io.mem.req_cmd <> mem_cmd_q.io.deq
io.mem.req_data <> mem_data_q.io.deq
} else { // Don't make the data buffers and try to flow cmd and data
io.mem.req_cmd.valid := Bool(false)
io.mem.req_data.valid := Bool(false)
io.mem.req_cmd.bits.rw := has_data
io.mem.req_cmd.bits.tag := tag_out
io.mem.req_cmd.bits.addr := addr_out
io.mem.req_data.bits.data := Mux(data_from_rel,
io.tl.release.bits.payload.data,
io.tl.acquire.bits.payload.data)
when(!active_out){
io.tl.release.ready := io.mem.req_data.ready
io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid
io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) ||
(io.tl.acquire.valid && acq_has_data)
when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) {
active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid
io.mem.req_cmd.valid := Bool(true)
cmd_sent_out := io.mem.req_cmd.ready
tag_out := io.mem.req_cmd.bits.tag
addr_out := io.mem.req_data.bits.data
has_data := io.mem.req_cmd.bits.rw
tl_done_out := tl_wrap_out
when(io.tl.release.valid) {
data_from_rel := Bool(true)
make_grant_ack := Bool(true)
io.mem.req_cmd.bits.rw := rel_has_data
io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id
io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block
io.mem.req_data.bits.data := io.tl.release.bits.payload.data
} .elsewhen(io.tl.acquire.valid) {
data_from_rel := Bool(false)
make_grant_ack := acq_has_data
io.mem.req_cmd.bits.rw := acq_has_data
io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id,
io.tl.acquire.bits.payload.a_type,
io.tl.acquire.bits.payload.is_builtin_type)
io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block
io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data
}
}
}
when(tl_wrap_out) { tl_done_out := Bool(true) }
when(mif_wrap_out) { mif_done_out := Bool(true) }
when(cmd_sent_out && (!has_data || mif_done_out)) {
active_out := Bool(false)
when(active_out) {
io.mem.req_cmd.valid := !cmd_sent_out
cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire()
when(has_data && !tl_done_out) {
when(data_from_rel) {
io.tl.release.ready := io.mem.req_data.ready
io.mem.req_data.valid := io.tl.release.valid
} .otherwise {
io.tl.acquire.ready := io.mem.req_data.ready
io.mem.req_data.valid := io.tl.acquire.valid
}
}
when(tl_wrap_out) { tl_done_out := Bool(true) }
when(tl_done_out && make_grant_ack) {
gnt_arb.io.in(1).valid := Bool(true)
when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) }
}
when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) {
active_out := Bool(false)
}
}
}
mem_cmd_q.io.enq.bits.rw := has_data
mem_cmd_q.io.enq.bits.tag := tag_out
mem_cmd_q.io.enq.bits.addr := addr_out
mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data
io.mem.req_cmd <> mem_cmd_q.io.deq
io.mem.req_data <> mem_data_q.io.deq
// Aggregate incoming MemIO responses into TL Grants
val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data
val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats)
val active_in = Reg(init=Bool(false))
val mif_done_in = Reg(init=Bool(false))
val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats)
val tag_in = Reg(UInt(width = mifTagBits))
val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) }
val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone }
tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits)
val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits)
val mif_prog_in = mif_cnt_in*UInt(mifDataBits)
when(!active_in) {
io.mem.resp.ready := Bool(true)
when(io.mem.resp.valid) {
active_in := Bool(true)
mif_done_in := mif_wrap_in
tag_in := io.mem.resp.bits.tag
mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data
}
}
if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) {
val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data
val mif_done_in = Reg(init=Bool(false))
val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) }
val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone }
tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits)
val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits)
val mif_prog_in = mif_cnt_in*UInt(mifDataBits)
gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = tag_in(0),
a_type = tag_in >> UInt(1),
client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = tl_buf_in(tl_cnt_in))
when(active_in) {
io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in
when(!mif_done_in) {
when(!active_in) {
io.mem.resp.ready := Bool(true)
when(io.mem.resp.valid) {
mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data
active_in := Bool(true)
mif_done_in := mif_wrap_in
tag_in := io.mem.resp.bits.tag
mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data
}
}
when(mif_wrap_in) { mif_done_in := Bool(true) }
when(tl_wrap_in) { active_in := Bool(false) }
when(active_in) {
gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in
when(!mif_done_in) {
io.mem.resp.ready := Bool(true)
when(io.mem.resp.valid) {
mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data
}
}
when(mif_wrap_in) { mif_done_in := Bool(true) }
when(tl_wrap_in) { active_in := Bool(false) }
}
} else { // Don't generate all the uneeded data buffers and flow resp
gnt_arb.io.in(0).valid := io.mem.resp.valid
io.mem.resp.ready := gnt_arb.io.in(0).ready
gnt_arb.io.in(0).bits.payload :=
ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = io.mem.resp.bits.tag(0),
a_type = io.mem.resp.bits.tag >> UInt(1),
client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = io.mem.resp.bits.data)
}
io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true),
g_type = Grant.uncachedReadBlock,
client_xact_id = tag_in,
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = tl_buf_in(tl_cnt_in))
}
class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module
@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul
dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw
}
class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module {
class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module {
val io = new Bundle {
val uncached = new UncachedTileLinkIO().flip
val tl = new TileLinkIO().flip
val mem = new MemPipeIO
}
val a = Module(new MemIOUncachedTileLinkIOConverter(2))
val a = Module(new MemIOTileLinkIOConverter(1))
val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles))
a.io.uncached <> io.uncached
a.io.tl <> io.tl
b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true)
b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true)
a.io.mem.resp <> b.io.cpu.resp

188
uncore/src/main/scala/metadata.scala Normal file
View File

@ -0,0 +1,188 @@
// See LICENSE for license details.
package uncore
import Chisel._
// Classes to represent coherence information in clients and managers
abstract class CoherenceMetadata extends Bundle {
val co = params(TLCoherencePolicy)
val id = params(TLId)
}
/* The ClientMetadata stores the client-side coherence information,
such as permissions on the data and whether the data is dirty.
Its API can be used to make TileLink messages in response to
memory operations or TileLink Probes.
*/
class ClientMetadata extends CoherenceMetadata {
val state = UInt(width = co.clientStateWidth)
def ===(rhs: ClientMetadata): Bool = this.state === rhs.state
def isValid(dummy: Int = 0): Bool = co.isValid(this)
def isHit(cmd: UInt): Bool = co.isHit(cmd, this)
def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this)
def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool =
co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this)
def requiresReleaseOnCacheControl(cmd: UInt): Bool =
co.requiresReleaseOnCacheControl(cmd: UInt, this)
def requiresVoluntaryWriteback(dummy: Int = 0): Bool =
co.requiresReleaseOnCacheControl(M_FLUSH, this)
def makeAcquire(
client_xact_id: UInt,
addr_block: UInt,
op_code: UInt): Acquire = {
Bundle(Acquire(
is_builtin_type = Bool(false),
a_type = co.getAcquireType(op_code, this),
client_xact_id = client_xact_id,
addr_block = addr_block,
union = Cat(op_code, Bool(true))),
{ case TLId => id })
}
def makeVoluntaryWriteback(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Bundle(Release(
voluntary = Bool(true),
r_type = co.getReleaseType(M_FLUSH, this),
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def makeRelease(
prb: Probe,
client_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Bundle(Release(
voluntary = Bool(false),
r_type = co.getReleaseType(prb, this),
client_xact_id = client_xact_id,
addr_block = prb.addr_block,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def onGrant(incoming: Grant, pending: UInt): ClientMetadata =
Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id })
def onProbe(incoming: Probe): ClientMetadata =
Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id })
def onHit(cmd: UInt): ClientMetadata =
Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id })
def onCacheControl(cmd: UInt): ClientMetadata =
Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id })
}
object ClientMetadata {
def apply(state: UInt) = {
val meta = new ClientMetadata
meta.state := state
meta
}
def onReset = new ClientMetadata().co.clientMetadataOnReset
}
/* The ManagerMetadata stores manager-side information about the status
of a cache block, including whether it has any known sharers. Its
API can be used to create Probe and Grant TileLink messages.
*/
class ManagerMetadata extends CoherenceMetadata {
// val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel
val sharers = UInt(width = co.dir.width)
def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel
this.sharers === rhs.sharers
def full(dummy: Int = 0) = co.dir.full(this.sharers)
def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this)
def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this)
def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool =
co.requiresProbes(M_FLUSH, this)
def makeProbe(cmd: UInt, addr_block: UInt): Probe =
Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id })
def makeProbe(acq: Acquire): Probe =
Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id })
def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe =
makeProbe(M_FLUSH, addr_block)
def makeGrant(rel: Release, manager_xact_id: UInt): Grant = {
Bundle(Grant(
is_builtin_type = Bool(true),
g_type = Grant.voluntaryAckType,
client_xact_id = rel.client_xact_id,
manager_xact_id = manager_xact_id), { case TLId => id })
}
def makeGrant(
acq: Acquire,
manager_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
Bundle(Grant(
is_builtin_type = acq.isBuiltInType(),
g_type = Mux(acq.isBuiltInType(),
acq.getBuiltInGrantType(),
co.getGrantType(acq, this)),
client_xact_id = acq.client_xact_id,
manager_xact_id = manager_xact_id,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def onRelease(incoming: Release, src: UInt): ManagerMetadata =
Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id })
def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata =
Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id })
}
object ManagerMetadata {
def apply(sharers: UInt, state: UInt = UInt(width = 0)) = {
val meta = new ManagerMetadata
//meta.state := state TODO: Fix 0-width wires in Chisel
meta.sharers := sharers
meta
}
def apply() = {
val meta = new ManagerMetadata
//meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel
meta.sharers := meta.co.dir.flush
meta
}
def onReset = new ManagerMetadata().co.managerMetadataOnReset
}
/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy
that is a Manager with respect to some inner caches and a Client with
respect to some outer cache.
*/
class HierarchicalMetadata extends CoherenceMetadata {
val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)})
val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)})
def ===(rhs: HierarchicalMetadata): Bool =
this.inner === rhs.inner && this.outer === rhs.outer
def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs)
}
object HierarchicalMetadata {
def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = {
val m = new HierarchicalMetadata
m.inner := inner
m.outer := outer
m
}
def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset)
}
case object InnerTLId extends Field[String]
case object OuterTLId extends Field[String]

307
uncore/src/main/scala/tilelink.scala
View File

@ -8,30 +8,35 @@ import scala.math.max
// external requirements or design space exploration
//
case object TLId extends Field[String] // Unique name per network
case object TLCoherence extends Field[CoherencePolicy]
case object TLAddrBits extends Field[Int]
case object TLCoherencePolicy extends Field[CoherencePolicy]
case object TLBlockAddrBits extends Field[Int]
case object TLManagerXactIdBits extends Field[Int]
case object TLClientXactIdBits extends Field[Int]
case object TLDataBits extends Field[Int]
case object TLDataBeats extends Field[Int]
case object TLNetworkIsOrderedP2P extends Field[Boolean]
abstract trait TileLinkParameters extends UsesParameters {
val tlBlockAddrBits = params(TLAddrBits)
val tlBlockAddrBits = params(TLBlockAddrBits)
val tlClientXactIdBits = params(TLClientXactIdBits)
val tlManagerXactIdBits = params(TLManagerXactIdBits)
val tlDataBits = params(TLDataBits)
val tlDataBeats = params(TLDataBeats)
val tlCoh = params(TLCoherencePolicy)
val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8
val tlBeatAddrBits = log2Up(tlDataBeats)
val tlByteAddrBits = log2Up(tlWriteMaskBits)
val tlAtomicOpcodeBits = M_SZ
val tlUncachedOperandSizeBits = MT_SZ
val tlSubblockUnionBits = max(tlWriteMaskBits,
val tlMemoryOpcodeBits = M_SZ
val tlMemoryOperandSizeBits = MT_SZ
val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes),
tlCoh.acquireTypeWidth)
val tlAcquireUnionBits = max(tlWriteMaskBits,
(tlByteAddrBits +
tlUncachedOperandSizeBits +
tlAtomicOpcodeBits)) + 1
val co = params(TLCoherence)
val networkPreservesPointToPointOrdering = false //TODO: check physical network type
tlMemoryOperandSizeBits +
tlMemoryOpcodeBits)) + 1
val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes),
tlCoh.grantTypeWidth) + 1
val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P)
}
abstract class TLBundle extends Bundle with TileLinkParameters
@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now
trait HasCacheBlockAddress extends TLBundle {
val addr_block = UInt(width = tlBlockAddrBits)
def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block
def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr
def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block
def conflicts(addr: UInt) = this.addr_block === addr
}
trait HasTileLinkBeatId extends TLBundle {
@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle {
val manager_xact_id = Bits(width = tlManagerXactIdBits)
}
abstract trait HasTileLinkData extends HasTileLinkBeatId {
trait HasTileLinkData extends HasTileLinkBeatId {
val data = UInt(width = tlDataBits)
def hasData(dummy: Int = 0): Bool
def hasMultibeatData(dummy: Int = 0): Bool
}
@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel
with HasClientTransactionId
with HasTileLinkData {
// Actual bundle fields
val builtin_type = Bool()
val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth))
val subblock = Bits(width = tlSubblockUnionBits)
val is_builtin_type = Bool()
val a_type = UInt(width = tlAcquireTypeBits)
val union = Bits(width = tlAcquireUnionBits)
// Utility funcs for accessing subblock union
val opSizeOff = tlByteAddrBits + 1
val opCodeOff = tlUncachedOperandSizeBits + opSizeOff
val opMSB = tlAtomicOpcodeBits + opCodeOff
def allocate(dummy: Int = 0) = subblock(0)
def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1)
def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff)
def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff)
def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1)
def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0))
val opCodeOff = 1
val opSizeOff = tlMemoryOpcodeBits + opCodeOff
val addrByteOff = tlMemoryOperandSizeBits + opSizeOff
val addrByteMSB = tlByteAddrBits + addrByteOff
def allocate(dummy: Int = 0) = union(0)
def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff))
def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff)
def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff)
def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1)
def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte())
// Other helper funcs
def is(t: UInt) = a_type === t
def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type)
def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type &&
def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type)
// Assumes no custom types have data
def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type)
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() &&
Acquire.typesWithMultibeatData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type)
def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType
def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe =
Probe(co.getProbeType(this, meta), this.addr_block)
def makeGrant(
manager_xact_id: UInt,
meta: ManagerMetadata = co.managerMetadataOnFlush,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
Grant(
builtin_type = this.builtin_type,
g_type = co.getGrantType(this, meta),
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id,
addr_beat = addr_beat,
data = data
)
def getBuiltInGrantType(dummy: Int = 0): UInt = {
MuxLookup(this.a_type, Grant.ackType, Array(
Acquire.getType -> Grant.dataBeatType,
Acquire.getBlockType -> Grant.dataBlockType,
Acquire.putType -> Grant.ackType,
Acquire.putBlockType -> Grant.ackType,
Acquire.putAtomicType -> Grant.dataBeatType))
}
}
object Acquire {
val nBuiltinAcquireTypes = 5
val nBuiltInTypes = 5
//TODO: Use Enum
def uncachedRead = UInt(0)
def uncachedReadBlock = UInt(1)
def uncachedWrite = UInt(2)
def uncachedWriteBlock = UInt(3)
def uncachedAtomic = UInt(4)
def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic)
def typesWithMultibeatData = Vec(uncachedWriteBlock)
def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock
def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock
def getType = UInt("b000")
def getBlockType = UInt("b001")
def putType = UInt("b010")
def putBlockType = UInt("b011")
def putAtomicType = UInt("b100")
def typesWithData = Vec(putType, putBlockType, putAtomicType)
def typesWithMultibeatData = Vec(putBlockType)
def typesOnSubBlocks = Vec(putType, getType, putAtomicType)
def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt
// Most generic constructor
def apply(
builtin_type: Bool,
is_builtin_type: Bool,
a_type: Bits,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0),
subblock: UInt = UInt(0)): Acquire = {
union: UInt = UInt(0)): Acquire = {
val acq = new Acquire
acq.builtin_type := builtin_type
acq.is_builtin_type := is_builtin_type
acq.a_type := a_type
acq.client_xact_id := client_xact_id
acq.addr_block := addr_block
acq.addr_beat := addr_beat
acq.data := data
acq.subblock := subblock
acq.union := union
acq
}
// For cached types
def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = {
apply(
builtin_type = Bool(false),
a_type = a_type,
client_xact_id = client_xact_id,
addr_block = addr_block)
}
// Copy constructor
def apply(a: Acquire): Acquire = {
val acq = new Acquire
@ -177,58 +169,73 @@ object Acquire {
}
// Asks for a single TileLink beat of data
object UncachedRead {
object Get {
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
builtin_type = Bool(true),
a_type = Acquire.uncachedRead,
is_builtin_type = Bool(true),
a_type = Acquire.getType,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
subblock = alloc)
union = Cat(M_XRD, alloc))
}
}
// Asks for an entire cache block of data
object UncachedReadBlock {
object GetBlock {
def apply(
client_xact_id: UInt = UInt(0),
addr_block: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
builtin_type = Bool(true),
a_type = Acquire.uncachedReadBlock,
is_builtin_type = Bool(true),
a_type = Acquire.getBlockType,
client_xact_id = client_xact_id,
addr_block = addr_block,
subblock = alloc.toUInt)
union = Cat(M_XRD, alloc))
}
}
object UncachedWrite {
// Writes up to a single TileLink beat of data, using mask
object Put {
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
data: UInt,
write_mask: UInt = Acquire.fullWriteMask,
alloc: Bool = Bool(true)): Acquire = {
write_mask: UInt = Acquire.fullWriteMask): Acquire = {
Acquire(
builtin_type = Bool(true),
a_type = Acquire.uncachedWrite,
is_builtin_type = Bool(true),
a_type = Acquire.putType,
addr_block = addr_block,
addr_beat = addr_beat,
client_xact_id = client_xact_id,
data = data,
subblock = Cat(write_mask, alloc))
union = Cat(write_mask, Bool(true)))
}
}
// For full block of data
object UncachedWriteBlock {
// Writes an entire cache block of data
object PutBlock {
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
data: UInt,
write_mask: UInt): Acquire = {
Acquire(
is_builtin_type = Bool(true),
a_type = Acquire.putBlockType,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
union = Cat(write_mask, (write_mask != Acquire.fullWriteMask)))
}
def apply(
client_xact_id: UInt,
addr_block: UInt,
@ -236,17 +243,18 @@ object UncachedWriteBlock {
data: UInt,
alloc: Bool = Bool(true)): Acquire = {
Acquire(
builtin_type = Bool(true),
a_type = Acquire.uncachedWriteBlock,
is_builtin_type = Bool(true),
a_type = Acquire.putBlockType,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
subblock = Cat(Acquire.fullWriteMask, alloc))
union = Cat(Acquire.fullWriteMask, alloc))
}
}
object UncachedAtomic {
// Performs an atomic operation in the outer memory
object PutAtomic {
def apply(
client_xact_id: UInt,
addr_block: UInt,
@ -256,48 +264,30 @@ object UncachedAtomic {
operand_size: UInt,
data: UInt): Acquire = {
Acquire(
builtin_type = Bool(true),
a_type = Acquire.uncachedAtomic,
is_builtin_type = Bool(true),
a_type = Acquire.putAtomicType,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true)))
union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true)))
}
}
class Probe extends ManagerToClientChannel
with HasCacheBlockAddress {
val p_type = UInt(width = co.probeTypeWidth)
val p_type = UInt(width = tlCoh.probeTypeWidth)
def is(t: UInt) = p_type === t
def makeRelease(
client_xact_id: UInt,
meta: ClientMetadata = co.clientMetadataOnFlush,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(false),
r_type = co.getReleaseType(this, meta),
client_xact_id = client_xact_id,
addr_block = this.addr_block,
addr_beat = addr_beat,
data = data)
}
}
object Probe {
val co = new Probe().co
def apply(p_type: UInt, addr_block: UInt) = {
val prb = new Probe
prb.p_type := p_type
prb.addr_block := addr_block
prb
}
def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = {
apply(co.getProbeType(M_FLUSH, meta), addr_block)
}
}
@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel
with HasCacheBlockAddress
with HasClientTransactionId
with HasTileLinkData {
val r_type = UInt(width = co.releaseTypeWidth)
val r_type = UInt(width = tlCoh.releaseTypeWidth)
val voluntary = Bool()
// Helper funcs
def is(t: UInt) = r_type === t
def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type)
def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type)
//TODO: Assumes all releases write back full cache blocks:
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type)
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type)
def isVoluntary(dummy: Int = 0) = voluntary
def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering)
def makeGrant(
manager_xact_id: UInt,
meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = {
Grant(
g_type = Grant.voluntaryAck,
builtin_type = Bool(true), // Grant.voluntaryAck is built-in type
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id
)
}
def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering)
}
object Release {
val co = new Release().co
def apply(
voluntary: Bool,
r_type: UInt,
@ -346,68 +324,52 @@ object Release {
rel.voluntary := voluntary
rel
}
def makeVoluntaryWriteback(
meta: ClientMetadata,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(true),
r_type = co.getReleaseType(M_FLUSH, meta),
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data)
}
}
class Grant extends ManagerToClientChannel
with HasTileLinkData
with HasClientTransactionId
with HasManagerTransactionId {
val builtin_type = Bool()
val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth))
val is_builtin_type = Bool()
val g_type = UInt(width = tlGrantTypeBits)
// Helper funcs
def is(t: UInt) = g_type === t
def hasData(dummy: Int = 0): Bool = Mux(builtin_type,
def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type
def is(t: UInt):Bool = g_type === t
def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(),
Grant.typesWithData.contains(g_type),
co.grantTypesWithData.contains(g_type))
tlCoh.grantTypesWithData.contains(g_type))
def hasMultibeatData(dummy: Int = 0): Bool =
Bool(tlDataBeats > 1) && Mux(builtin_type,
Bool(tlDataBeats > 1) && Mux(isBuiltInType(),
Grant.typesWithMultibeatData.contains(g_type),
co.grantTypesWithData.contains(g_type))
def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck)
def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary()
tlCoh.grantTypesWithData.contains(g_type))
def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType)
def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary()
def makeFinish(dummy: Int = 0): Finish = {
val f = new Finish
val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits })
f.manager_xact_id := this.manager_xact_id
f
}
}
object Grant {
val nBuiltinGrantTypes = 5
//TODO Use Enum
def voluntaryAck = UInt(0)
def uncachedRead = UInt(1)
def uncachedReadBlock = UInt(2)
def uncachedWrite = UInt(3)
def uncachedAtomic = UInt(4)
def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic)
def typesWithMultibeatData= Vec(uncachedReadBlock)
val nBuiltInTypes = 4
def voluntaryAckType = UInt("b00")
def ackType = UInt("b01")
def dataBeatType = UInt("b10")
def dataBlockType = UInt("b11")
def typesWithData = Vec(dataBlockType, dataBeatType)
def typesWithMultibeatData= Vec(dataBlockType)
def apply(
builtin_type: Bool,
is_builtin_type: Bool,
g_type: UInt,
client_xact_id: UInt,
manager_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
val gnt = new Grant
gnt.builtin_type := builtin_type
gnt.is_builtin_type := is_builtin_type
gnt.g_type := g_type
gnt.client_xact_id := client_xact_id
gnt.manager_xact_id := manager_xact_id
@ -415,23 +377,12 @@ object Grant {
gnt.data := data
gnt
}
def getGrantTypeForUncached(a: Acquire): UInt = {
MuxLookup(a.a_type, Grant.uncachedRead, Array(
Acquire.uncachedRead -> Grant.uncachedRead,
Acquire.uncachedReadBlock -> Grant.uncachedReadBlock,
Acquire.uncachedWrite -> Grant.uncachedWrite,
Acquire.uncachedWriteBlock -> Grant.uncachedWrite,
Acquire.uncachedAtomic -> Grant.uncachedAtomic
))
}
}
class Finish extends ClientToManagerChannel with HasManagerTransactionId
// Complete IO definitions for two types of TileLink clients
class UncachedTileLinkIO extends Bundle {
class UncachedTileLinkIO extends TLBundle {
val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire))
val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip
val finish = new DecoupledIO(new LogicalNetworkIO(new Finish))
@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule {
io.out.release.valid := Bool(false)
}
object TileLinkIOWrapper {
def apply[T <: Data](utl: UncachedTileLinkIO) = {
def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = {
val conv = Module(new TileLinkIOWrapper)(p)
conv.io.in <> utl
conv.io.out
}
def apply(utl: UncachedTileLinkIO): TileLinkIO = {
val conv = Module(new TileLinkIOWrapper)
conv.io.in <> utl
conv.io.out
}
def apply(tl: TileLinkIO) = tl
}
abstract trait HasArbiterTypes {

508
uncore/src/main/scala/uncore.scala
View File

@ -2,437 +2,147 @@
package uncore
import Chisel._
import scala.reflect.ClassTag
case object NReleaseTransactors extends Field[Int]
case object NProbeTransactors extends Field[Int]
case object NAcquireTransactors extends Field[Int]
case object NIncoherentClients extends Field[Int]
case object NCoherentClients extends Field[Int]
case object L2StoreDataQueueDepth extends Field[Int]
case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy]
case object L2DirectoryRepresentation extends Field[DirectoryRepresentation]
case object L2CoherencePolicy extends Field[CoherencePolicy]
abstract trait CoherenceAgentParameters extends UsesParameters
with TileLinkParameters {
trait CoherenceAgentParameters extends UsesParameters {
val nReleaseTransactors = 1
val nAcquireTransactors = params(NAcquireTransactors)
val nTransactors = nReleaseTransactors + nAcquireTransactors
val nCoherentClients = params(NCoherentClients)
val nIncoherentClients = params(NIncoherentClients)
val nClients = nCoherentClients + nIncoherentClients
val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)})
val outerDataBeats = outerTLParams(TLDataBeats)
val outerDataBits = outerTLParams(TLDataBits)
def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)})
val innerDataBeats = innerTLParams(TLDataBeats)
val innerDataBits = innerTLParams(TLDataBits)
val innerBeatAddrBits = log2Up(innerDataBeats)
val innerByteAddrBits = log2Up(innerDataBits/8)
require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement
}
abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters
abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters
class DataQueueLocation extends Bundle with CoherenceAgentParameters {
val idx = UInt(width = dqIdxBits)
val loc = UInt(width = log2Ceil(nDataQueueLocations))
}
object DataQueueLocation {
def apply(idx: UInt, loc: UInt) = {
val d = new DataQueueLocation
d.idx := idx
d.loc := loc
d
trait HasCoherenceAgentWiringHelpers {
def doOutputArbitration[T <: Data : ClassTag](
out: DecoupledIO[T],
ins: Seq[DecoupledIO[T]]) {
val arb = Module(new RRArbiter(out.bits.clone, ins.size))
out <> arb.io.out
arb.io.in zip ins map { case (a, in) => a <> in }
}
def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag](
out: DecoupledIO[S],
ins: Seq[DecoupledIO[S]]) {
def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData()
val arb = Module(new LockingRRArbiter(
out.bits.clone,
ins.size,
out.bits.payload.tlDataBeats,
lock _))
out <> arb.io.out
arb.io.in zip ins map { case (a, in) => a <> in }
}
def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) {
val idx = in.bits.id
outs.map(_.bits := in.bits)
outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
}
def doInputRouting[T <: HasManagerTransactionId](
in: DecoupledIO[LogicalNetworkIO[T]],
outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) {
val idx = in.bits.payload.manager_xact_id
outs.map(_.bits := in.bits)
outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
in.ready := Vec(outs.map(_.ready)).read(idx)
}
}
abstract class CoherenceAgent(innerId: String, outerId: String) extends Module
with CoherenceAgentParameters {
val io = new Bundle {
val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip
val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId})
val incoherent = Vec.fill(nClients){Bool()}.asInput
}
trait HasInnerTLIO extends CoherenceAgentBundle {
val inner = Bundle(new TileLinkIO)(innerTLParams).flip
val incoherent = Vec.fill(nClients){Bool()}.asInput
def iacq(dummy: Int = 0) = inner.acquire.bits.payload
def iprb(dummy: Int = 0) = inner.probe.bits.payload
def irel(dummy: Int = 0) = inner.release.bits.payload
def ignt(dummy: Int = 0) = inner.grant.bits.payload
def ifin(dummy: Int = 0) = inner.finish.bits.payload
}
class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends
CoherenceAgent(innerId, outerId) {
val internalDataBits = new DataQueueLocation().getWidth
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
// Create SHRs for outstanding transactions
val trackerList = (0 until nReleaseTransactors).map(id =>
Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++
(nReleaseTransactors until nTransactors).map(id =>
Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits}))
// Propagate incoherence flags
trackerList.map(_.io.tile_incoherent := io.incoherent.toBits)
// Queue to store impending UncachedWrite data
val acquire = io.inner.acquire
val sdq_val = Reg(init=Bits(0, sdqDepth))
val sdq_alloc_id = PriorityEncoder(~sdq_val)
val sdq_rdy = !sdq_val.andR
val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
// Handle acquire transaction initiation
val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
val block_acquires = any_acquire_conflict
val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
alloc_arb.io.in(i).valid := t.acquire.ready
t.acquire.bits := acquire.bits
t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
t.acquire.valid := alloc_arb.io.in(i).ready
}
acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
// Queue to store impending Voluntary Release data
val release = io.inner.release
val voluntary = release.bits.payload.isVoluntary()
val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width
val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1
when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
// Handle releases, which might be voluntary and might have data
val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b}
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
t.release.bits := release.bits
t.release.bits.payload.data := (if (i < nReleaseTransactors)
DataQueueLocation(rel_data_cnt, inVolWBQueue)
else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits
t.release.valid := release.valid && (release_idx === UInt(i))
}
release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx)
// Wire finished transaction acks
val ack = io.inner.finish
trackerList.map(_.io.inner.finish.valid := ack.valid)
trackerList.map(_.io.inner.finish.bits := ack.bits)
ack.ready := Bool(true)
// Wire probe requests to clients
val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size))
io.inner.probe <> probe_arb.io.out
probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe }
// Wire grant reply to initiating client
def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData()
val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _)))
io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat
io.inner.grant <> grant_arb.io.out
grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant }
// Create an arbiter for the one memory port
val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
{ case TLId => outerId; case TLDataBits => internalDataBits })
outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer }
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.payload.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
io.outer <> outer_arb.io.out
// Update SDQ valid bits
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
trait HasUncachedOuterTLIO extends CoherenceAgentBundle {
val outer = Bundle(new UncachedTileLinkIO)(outerTLParams)
def oacq(dummy: Int = 0) = outer.acquire.bits.payload
def ognt(dummy: Int = 0) = outer.grant.bits.payload
def ofin(dummy: Int = 0) = outer.finish.bits.payload
}
abstract class XactTracker(innerId: String, outerId: String) extends Module {
val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats))
val nClients = params(NCoherentClients) + params(NIncoherentClients)
val io = new Bundle {
val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip
val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId})
val tile_incoherent = Bits(INPUT, nClients)
val has_acquire_conflict = Bool(OUTPUT)
val has_release_match = Bool(OUTPUT)
}
val cacq = io.inner.acquire.bits
val crel = io.inner.release.bits
val cgnt = io.inner.grant.bits
val cfin = io.inner.finish.bits
val macq = io.outer.acquire.bits
val mgnt = io.outer.grant.bits
trait HasCachedOuterTLIO extends CoherenceAgentBundle {
val outer = Bundle(new TileLinkIO)(outerTLParams)
def oacq(dummy: Int = 0) = outer.acquire.bits.payload
def oprb(dummy: Int = 0) = outer.probe.bits.payload
def orel(dummy: Int = 0) = outer.release.bits.payload
def ognt(dummy: Int = 0) = outer.grant.bits.payload
def ofin(dummy: Int = 0) = outer.finish.bits.payload
}
class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
val state = Reg(init=s_idle)
class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO
val xact_src = Reg(io.inner.release.bits.header.src.clone)
val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone)
val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
val xact = Release(
voluntary = Bool(true),
r_type = xact_r_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block)
val collect_inner_data = Reg(init=Bool(false))
// TODO: assert that all releases have full blocks of data
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats)
val (outer_data_cnt, outer_data_done) =
Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats)
io.has_acquire_conflict := Bool(false)
io.has_release_match := crel.payload.isVoluntary()
io.outer.grant.ready := Bool(false)
io.outer.acquire.valid := Bool(false)
io.inner.acquire.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.grant.valid := Bool(false)
io.inner.finish.ready := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId))
io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
when(collect_inner_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
xact_data(inner_data_cnt) := crel.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
}
switch (state) {
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := crel.header.src
xact_r_type := crel.payload.r_type
xact_addr_block := crel.payload.addr_block
xact_client_xact_id := crel.payload.client_xact_id
xact_data(UInt(0)) := crel.payload.data
collect_inner_data := crel.payload.hasMultibeatData()
state := Mux(crel.payload.hasData(), s_outer,
Mux(crel.payload.requiresAck(), s_ack, s_idle))
}
}
is(s_outer) {
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
when(outer_data_done) {
state := Mux(xact.requiresAck(), s_grant, s_idle)
}
}
is(s_grant) {
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) {
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
abstract class CoherenceAgent extends CoherenceAgentModule {
def innerTL: TileLinkIO
def outerTL: TileLinkIO
def incoherent: Vec[Bool]
}
class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7)
val state = Reg(init=s_idle)
abstract class ManagerCoherenceAgent extends CoherenceAgent
with HasCoherenceAgentWiringHelpers {
val io = new ManagerTLIO
def innerTL = io.inner
def outerTL = TileLinkIOWrapper(io.outer, outerTLParams)
def incoherent = io.incoherent
}
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone)
val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone)
val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
val xact = Acquire(
builtin_type = xact_builtin_type,
a_type = xact_a_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block,
addr_beat = xact_addr_beat,
data = UInt(0),
subblock = xact_subblock)
class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO
val collect_inner_data = Reg(init=Bool(false))
//TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats)
val (outer_data_cnt, outer_data_done) =
Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats)
val (cgnt_data_cnt, cgnt_data_done) =
Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats)
abstract class HierarchicalCoherenceAgent extends CoherenceAgent {
val io = new HierarchicalTLIO
def innerTL = io.inner
def outerTL = io.outer
def incoherent = io.incoherent
}
trait HasTrackerConflictIO extends Bundle {
val has_acquire_conflict = Bool(OUTPUT)
val has_acquire_match = Bool(OUTPUT)
val has_release_match = Bool(OUTPUT)
}
val release_count = Reg(init=UInt(0, width = log2Up(nClients)))
val probe_flags = Reg(init=Bits(0, width = nClients))
val curr_p_id = PriorityEncoder(probe_flags)
class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO
class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO
val pending_outer_write = xact.hasData()
val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush)
val probe_initial_flags = Bits(width = nClients)
probe_initial_flags := Bits(0)
// issue self-probes for uncached read xacts to facilitate I$ coherence
val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe()
val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0)))
probe_initial_flags := ~(io.tile_incoherent | myflag)
io.has_acquire_conflict := xact.conflicts(cacq.payload) &&
!collect_inner_data &&
(state != s_idle)
io.has_release_match := xact.conflicts(crel.payload) &&
!crel.payload.isVoluntary() &&
(state != s_idle)
val outer_write_acq = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
val outer_write_rel = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = crel.payload.addr_beat,
data = crel.payload.data),
{ case TLId => outerId })
val outer_read = Bundle(UncachedReadBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block),
{ case TLId => outerId })
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
io.outer.grant.ready := io.inner.grant.ready
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := xact.makeProbe()
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
when(collect_inner_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_data(inner_data_cnt) := cacq.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
abstract class XactTracker extends CoherenceAgentModule {
def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = {
val multi = data.hasMultibeatData()
val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats)
val cnt = Mux(multi, multi_cnt, beat)
val done = Mux(multi, multi_done, inc)
(cnt, done)
}
switch (state) {
is(s_idle) {
io.inner.acquire.ready := Bool(true)
val needs_outer_write = cacq.payload.hasData()
val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush)
when(io.inner.acquire.valid) {
xact_builtin_type := cacq.payload.builtin_type
xact_a_type := cacq.payload.a_type
xact_addr_block := cacq.payload.addr_block
xact_addr_beat := cacq.payload.addr_beat
xact_client_xact_id := cacq.payload.client_xact_id
xact_data(UInt(0)) := cacq.payload.data
xact_subblock := cacq.payload.subblock
xact_src := cacq.header.src
collect_inner_data := cacq.payload.hasMultibeatData()
probe_flags := probe_initial_flags
release_count := PopCount(probe_initial_flags)
state := Mux(probe_initial_flags.orR, s_probe,
Mux(needs_outer_write, s_mem_write,
Mux(needs_outer_read, s_mem_read, s_make_grant)))
}
}
is(s_probe) {
// Generate probes
io.inner.probe.valid := probe_flags.orR
when(io.inner.probe.ready) {
probe_flags := probe_flags & ~(UIntToOH(curr_p_id))
}
// Handle releases, which may have data to be written back
io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready
when(io.inner.release.valid) {
when(crel.payload.hasData()) {
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_write_rel
when(io.outer.acquire.ready) {
when(outer_data_done) {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
} .otherwise {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
}
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_read
when(io.outer.acquire.ready) { state := s_mem_resp }
}
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
io.outer.acquire.bits.payload := outer_write_acq
when(outer_data_done) {
state := Mux(pending_outer_read, s_mem_read, s_make_grant)
}
}
is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_mem_resp) { // Wait to forward grants from outer memory
when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) {
io.inner.grant.valid := Bool(true)
}
when(cgnt_data_done) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) {
state := s_idle
}
}
def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](
in: DecoupledIO[LogicalNetworkIO[T]],
beat: UInt = UInt(0)) = {
connectDataBeatCounter(in.fire(), in.bits.payload, beat)
}
def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = {
connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2
}
}