riscv/rocket-chip
1
0
Fork 0
Code Releases Activity

New metadata-based coherence API

Browse Source
This commit is contained in:
Henry Cook 2015-02-28 17:02:13 -08:00
parent 0a8722e881
commit 1bed6ea498
9 changed files with 2117 additions and 1851 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

380
uncore/src/main/scala/broadcast.scala Normal file
View File

@ -0,0 +1,380 @@
// See LICENSE for license details.
package uncore
import Chisel._
case object L2StoreDataQueueDepth extends Field[Int]
trait BroadcastHubParameters extends CoherenceAgentParameters {
val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
}
class DataQueueLocation extends Bundle with BroadcastHubParameters {
val idx = UInt(width = dqIdxBits)
val loc = UInt(width = log2Ceil(nDataQueueLocations))
}
object DataQueueLocation {
def apply(idx: UInt, loc: UInt) = {
val d = new DataQueueLocation
d.idx := idx
d.loc := loc
d
}
}
class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent
with BroadcastHubParameters {
val internalDataBits = new DataQueueLocation().getWidth
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
// Create SHRs for outstanding transactions
val trackerList = (0 until nReleaseTransactors).map(id =>
Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++
(nReleaseTransactors until nTransactors).map(id =>
Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits}))
// Propagate incoherence flags
trackerList.map(_.io.incoherent := io.incoherent.toBits)
// Queue to store impending Put data
val acquire = io.inner.acquire
val sdq_val = Reg(init=Bits(0, sdqDepth))
val sdq_alloc_id = PriorityEncoder(~sdq_val)
val sdq_rdy = !sdq_val.andR
val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
// Handle acquire transaction initiation
val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
val block_acquires = any_acquire_conflict
val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
alloc_arb.io.in(i).valid := t.acquire.ready
t.acquire.bits := acquire.bits
t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
t.acquire.valid := alloc_arb.io.in(i).ready
}
acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
// Queue to store impending Voluntary Release data
val release = io.inner.release
val voluntary = release.bits.payload.isVoluntary()
val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1
when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
// Handle releases, which might be voluntary and might have data
val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b}
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
t.release.bits := release.bits
t.release.bits.payload.data := (if (i < nReleaseTransactors)
DataQueueLocation(rel_data_cnt, inVolWBQueue)
else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits
t.release.valid := release.valid && (release_idx === UInt(i))
}
release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx)
// Wire finished transaction acks
val ack = io.inner.finish
trackerList.map(_.io.inner.finish.valid := ack.valid)
trackerList.map(_.io.inner.finish.bits := ack.bits)
ack.ready := Bool(true)
// Wire probe requests and grant reply to clients, finish acks from clients
// Note that we bypass the Grant data subbundles
io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
// Create an arbiter for the one memory port
val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
{ case TLId => params(OuterTLId)
case TLDataBits => internalDataBits })
outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer }
// Get the pending data out of the store data queue
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.payload.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
io.outer <> outer_arb.io.out
// Update SDQ valid bits
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
}
class BroadcastXactTracker extends XactTracker {
val io = new ManagerXactTrackerIO
}
class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker {
val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.release.bits.header.src.clone)
val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 }))
val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) }
val coh = ManagerMetadata.onReset
val collect_irel_data = Reg(init=Bool(false))
val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats))
val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
io.has_acquire_conflict := Bool(false)
io.has_release_match := io.irel().isVoluntary()
io.has_acquire_match := Bool(false)
io.outer.acquire.valid := Bool(false)
io.outer.grant.ready := Bool(false)
io.outer.finish.valid := Bool(false)
io.inner.acquire.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.grant.valid := Bool(false)
io.inner.finish.ready := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId))
//TODO: Use io.outer.release instead?
io.outer.acquire.bits.payload := Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = oacq_data_cnt,
data = data_buffer(oacq_data_cnt)))(outerTLParams)
when(collect_irel_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
data_buffer(io.irel().addr_beat) := io.irel().data
irel_data_valid(io.irel().addr_beat) := Bool(true)
}
when(irel_data_done) { collect_irel_data := Bool(false) }
}
switch (state) {
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := io.inner.release.bits.header.src
xact := io.irel()
data_buffer(UInt(0)) := io.irel().data
collect_irel_data := io.irel().hasMultibeatData()
irel_data_valid := io.irel().hasData() << io.irel().addr_beat
state := Mux(io.irel().hasData(), s_outer,
Mux(io.irel().requiresAck(), s_ack, s_idle))
}
}
is(s_outer) {
io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt)
when(oacq_data_done) {
state := Mux(xact.requiresAck(), s_grant, s_idle)
}
}
is(s_grant) { // Forward the Grant.voluntaryAck
io.outer.grant.ready := io.inner.grant.ready
io.inner.grant.valid := io.outer.grant.valid
when(io.inner.grant.fire()) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_ack) {
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
}
class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker {
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 }))
val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) }
val coh = ManagerMetadata.onReset
assert(!(state != s_idle && xact.isBuiltInType() &&
Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)),
"Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO
val release_count = Reg(init=UInt(0, width = log2Up(nClients)))
val probe_flags = Reg(init=Bits(0, width = nClients))
val curr_p_id = PriorityEncoder(probe_flags)
val probe_initial_flags = Bits(width = nClients)
probe_initial_flags := Bits(0)
// issue self-probes for uncached read xacts to facilitate I$ coherence
val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe()
val myflag = Mux(probe_self, Bits(0),
UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0)))
probe_initial_flags := ~(io.incoherent.toBits | myflag)
val collect_iacq_data = Reg(init=Bool(false))
val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats))
val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire)
val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant)
val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
val pending_ognt_ack = Reg(init=Bool(false))
val pending_outer_write = xact.hasData()
val pending_outer_read = io.ignt().hasData()
val pending_outer_write_ = io.iacq().hasData()
val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData()
io.has_acquire_conflict := xact.conflicts(io.iacq()) &&
!collect_iacq_data &&
(state != s_idle)
io.has_release_match := xact.conflicts(io.irel()) &&
!io.irel().isVoluntary() &&
(state != s_idle)
io.has_acquire_match := Bool(false)
val outer_write_acq = Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = oacq_data_cnt,
data = data_buffer(oacq_data_cnt)))(outerTLParams)
val outer_write_rel = Bundle(PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = io.irel().addr_beat,
data = io.irel().data))(outerTLParams)
val outer_read = Bundle(GetBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block))(outerTLParams)
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
io.outer.grant.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := coh.makeProbe(xact)
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.finish.ready := Bool(false)
when(collect_iacq_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
data_buffer(io.iacq().addr_beat) := io.iacq().data
iacq_data_valid(io.iacq().addr_beat) := Bool(true)
}
when(iacq_data_done) { collect_iacq_data := Bool(false) }
}
when(pending_ognt_ack) {
io.outer.grant.ready := Bool(true)
when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) }
//TODO add finish queue if this isnt the last level manager
}
switch (state) {
is(s_idle) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_src := io.inner.acquire.bits.header.src
xact := io.iacq()
data_buffer(UInt(0)) := io.iacq().data
collect_iacq_data := io.iacq().hasMultibeatData()
iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat
probe_flags := probe_initial_flags
release_count := PopCount(probe_initial_flags)
state := Mux(probe_initial_flags.orR, s_probe,
Mux(pending_outer_write_, s_mem_write,
Mux(pending_outer_read_, s_mem_read, s_make_grant)))
}
}
is(s_probe) {
// Generate probes
io.inner.probe.valid := probe_flags.orR
when(io.inner.probe.ready) {
probe_flags := probe_flags & ~(UIntToOH(curr_p_id))
}
// Handle releases, which may have data to be written back
io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready
when(io.inner.release.valid) {
when(io.irel().hasData()) {
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_write_rel
when(io.outer.acquire.ready) {
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
} .otherwise {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
}
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt)
io.outer.acquire.bits.payload := outer_write_acq
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
state := Mux(pending_outer_read, s_mem_read, s_mem_resp)
}
}
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := !pending_ognt_ack
io.outer.acquire.bits.payload := outer_read
when(io.outer.acquire.fire()) { state := s_mem_resp }
}
is(s_mem_resp) { // Wait to forward grants from outer memory
io.outer.grant.ready := io.inner.grant.ready
io.inner.grant.valid := io.outer.grant.valid
when(ignt_data_done) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
}

1232
uncore/src/main/scala/cache.scala
View File

File diff suppressed because it is too large Load Diff

985
uncore/src/main/scala/coherence.scala
View File

File diff suppressed because it is too large Load Diff

43
uncore/src/main/scala/directory.scala Normal file
View File

@ -0,0 +1,43 @@
// See LICENSE for license details.
package uncore
import Chisel._
// This class encapsulates transformations on different directory information
// storage formats
abstract class DirectoryRepresentation(val width: Int) {
def pop(prev: UInt, id: UInt): UInt
def push(prev: UInt, id: UInt): UInt
def flush: UInt
def none(s: UInt): Bool
def one(s: UInt): Bool
def count(s: UInt): UInt
def next(s: UInt): UInt
def full(s: UInt): UInt
}
abstract trait HasDirectoryRepresentation {
val dir: DirectoryRepresentation
}
class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) {
def pop(prev: UInt, id: UInt) = UInt(0)
def push(prev: UInt, id: UInt) = UInt(0)
def flush = UInt(0)
def none(s: UInt) = Bool(false)
def one(s: UInt) = Bool(false)
def count(s: UInt) = UInt(nClients)
def next(s: UInt) = UInt(0)
def full(s: UInt) = SInt(-1, width = nClients).toUInt
}
class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) {
def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id)
def push(prev: UInt, id: UInt) = prev | UIntToOH(id)
def flush = UInt(0, width = width)
def none(s: UInt) = s === UInt(0)
def one(s: UInt) = PopCount(s) === UInt(1)
def count(s: UInt) = PopCount(s)
def next(s: UInt) = PriorityEncoder(s)
def full(s: UInt) = s
}

7
uncore/src/main/scala/htif.scala
View File

@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int]
abstract trait HTIFParameters extends UsesParameters { abstract trait HTIFParameters extends UsesParameters {
val dataBits = params(TLDataBits) val dataBits = params(TLDataBits)
val dataBeats = params(TLDataBeats) val dataBeats = params(TLDataBeats)
val co = params(TLCoherence)
val w = params(HTIFWidth) val w = params(HTIFWidth)
val nSCR = params(HTIFNSCR) val nSCR = params(HTIFNSCR)
val offsetBits = params(HTIFOffsetBits) val offsetBits = params(HTIFOffsetBits)
@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
val init_addr = addr.toUInt >> UInt(offsetBits-3) val init_addr = addr.toUInt >> UInt(offsetBits-3)
io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq
io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem,
UncachedWriteBlock( PutBlock(
addr_block = init_addr, addr_block = init_addr,
addr_beat = cnt, addr_beat = cnt,
client_xact_id = UInt(0), client_xact_id = UInt(0),
data = mem_req_data), data = mem_req_data),
UncachedReadBlock(addr_block = init_addr)) GetBlock(addr_block = init_addr))
io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.payload.data := mem_req_data
io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack
io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.payload.manager_xact_id := mem_gxid
@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
for (i <- 0 until scr_rdata.size) for (i <- 0 until scr_rdata.size)
scr_rdata(i) := io.scr.rdata(i) scr_rdata(i) := io.scr.rdata(i)
scr_rdata(0) := UInt(nCores) scr_rdata(0) := UInt(nCores)
scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20)
io.scr.wen := Bool(false) io.scr.wen := Bool(false)
io.scr.wdata := pcr_wdata io.scr.wdata := pcr_wdata

312
uncore/src/main/scala/memserdes.scala
View File

@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side
io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w))
} }
//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary
//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO
class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { class MemIOTileLinkIOConverter(qDepth: Int) extends Module {
val io = new Bundle { val io = new Bundle {
val uncached = new UncachedTileLinkIO().flip val tl = new TileLinkIO().flip
val mem = new MemIO val mem = new MemIO
} }
val co = params(TLCoherence)
val mifTagBits = params(MIFTagBits) val mifTagBits = params(MIFTagBits)
val mifDataBits = params(MIFDataBits) val mifDataBits = params(MIFDataBits)
val mifDataBeats = params(MIFDataBeats) val mifDataBeats = params(MIFDataBeats)
@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
val tlDataBeats = params(TLDataBeats) val tlDataBeats = params(TLDataBeats)
val dataBits = tlDataBits*tlDataBeats val dataBits = tlDataBits*tlDataBeats
require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats)
//require(params(TLClientXactIdBits) <= params(MIFTagBits)) require(params(TLClientXactIdBits) <= params(MIFTagBits))
io.tl.acquire.ready := Bool(false)
io.tl.probe.valid := Bool(false)
io.tl.release.ready := Bool(false)
io.tl.finish.ready := Bool(true)
io.mem.resp.ready := Bool(false)
val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2))
io.tl.grant <> gnt_arb.io.out
val acq_has_data = io.tl.acquire.bits.payload.hasData()
val rel_has_data = io.tl.release.bits.payload.hasData()
// Decompose outgoing TL Acquires into MemIO cmd and data // Decompose outgoing TL Acquires into MemIO cmd and data
val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth))
val mem_data_q = Module(new Queue(new MemData, qDepth))
io.uncached.acquire.ready := Bool(false)
io.uncached.grant.valid := Bool(false)
io.mem.resp.ready := Bool(false)
mem_cmd_q.io.enq.valid := Bool(false)
mem_data_q.io.enq.valid := Bool(false)
//TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are
//acceptable Acquire types
val acq_has_data = io.uncached.acquire.bits.payload.hasData()
val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats)
val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats)
val active_out = Reg(init=Bool(false)) val active_out = Reg(init=Bool(false))
val cmd_sent_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false))
val tl_done_out = Reg(init=Bool(false))
val mif_done_out = Reg(init=Bool(false))
val tag_out = Reg(Bits()) val tag_out = Reg(Bits())
val addr_out = Reg(Bits()) val addr_out = Reg(Bits())
val has_data = Reg(init=Bool(false)) val has_data = Reg(init=Bool(false))
val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } val data_from_rel = Reg(init=Bool(false))
val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } val (tl_cnt_out, tl_wrap_out) =
mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) Counter((io.tl.acquire.fire() && acq_has_data) ||
val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) (io.tl.release.fire() && rel_has_data), tlDataBeats)
val tl_prog_out = tl_cnt_out*UInt(tlDataBits) val tl_done_out = Reg(init=Bool(false))
val make_grant_ack = Reg(init=Bool(false))
val grant_for_rel = Grant(
is_builtin_type = Bool(true),
g_type = Grant.voluntaryAckType,
client_xact_id = tag_out,
manager_xact_id = UInt(0))
val grant_for_acq_write = ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = tag_out(0),
a_type = tag_out >> UInt(1),
client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0))
gnt_arb.io.in(1).valid := Bool(false)
gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write)
when(!active_out){ if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) {
io.uncached.acquire.ready := Bool(true) val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth))
when(io.uncached.acquire.valid) { val mem_data_q = Module(new Queue(new MemData, qDepth))
active_out := Bool(true) mem_cmd_q.io.enq.valid := Bool(false)
cmd_sent_out := Bool(false) mem_data_q.io.enq.valid := Bool(false)
tag_out := io.uncached.acquire.bits.payload.client_xact_id val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats)
addr_out := io.uncached.acquire.bits.payload.addr_block val mif_done_out = Reg(init=Bool(false))
has_data := acq_has_data val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) }
tl_done_out := tl_wrap_out val mif_buf_out = Vec.fill(mifDataBeats){ new MemData }
mif_done_out := Bool(false) mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits)
tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits)
val tl_prog_out = tl_cnt_out*UInt(tlDataBits)
when(!active_out){
io.tl.release.ready := Bool(true)
io.tl.acquire.ready := !io.tl.release.valid
when(io.tl.release.valid) {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := io.tl.release.bits.payload.client_xact_id
addr_out := io.tl.release.bits.payload.addr_block
has_data := rel_has_data
data_from_rel := Bool(true)
make_grant_ack := Bool(true)
tl_done_out := tl_wrap_out
tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data
} .elsewhen(io.tl.acquire.valid) {
active_out := Bool(true)
cmd_sent_out := Bool(false)
tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id,
io.tl.acquire.bits.payload.a_type,
io.tl.acquire.bits.payload.is_builtin_type)
addr_out := io.tl.acquire.bits.payload.addr_block
has_data := acq_has_data
data_from_rel := Bool(false)
make_grant_ack := acq_has_data
tl_done_out := tl_wrap_out
tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data
}
} }
} when(active_out) {
when(active_out) { mem_cmd_q.io.enq.valid := !cmd_sent_out
when(!cmd_sent_out) { cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire()
mem_cmd_q.io.enq.valid := Bool(true) when(has_data) {
} when(!tl_done_out) {
when(has_data) { io.tl.acquire.ready := Bool(true)
when(!tl_done_out) { when(io.tl.acquire.valid) {
io.uncached.acquire.ready := Bool(true) tl_buf_out(tl_cnt_out) := Mux(data_from_rel,
when(io.uncached.acquire.valid) { io.tl.release.bits.payload.data,
tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data io.tl.acquire.bits.payload.data)
}
}
when(!mif_done_out) {
mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out
} }
} }
when(!mif_done_out) { when(tl_wrap_out) { tl_done_out := Bool(true) }
mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out when(mif_wrap_out) { mif_done_out := Bool(true) }
when(tl_done_out && make_grant_ack) {
gnt_arb.io.in(1).valid := Bool(true)
when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) }
}
when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) {
active_out := Bool(false)
} }
} }
when(mem_cmd_q.io.enq.fire()) {
cmd_sent_out := Bool(true) mem_cmd_q.io.enq.bits.rw := has_data
mem_cmd_q.io.enq.bits.tag := tag_out
mem_cmd_q.io.enq.bits.addr := addr_out
mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data
io.mem.req_cmd <> mem_cmd_q.io.deq
io.mem.req_data <> mem_data_q.io.deq
} else { // Don't make the data buffers and try to flow cmd and data
io.mem.req_cmd.valid := Bool(false)
io.mem.req_data.valid := Bool(false)
io.mem.req_cmd.bits.rw := has_data
io.mem.req_cmd.bits.tag := tag_out
io.mem.req_cmd.bits.addr := addr_out
io.mem.req_data.bits.data := Mux(data_from_rel,
io.tl.release.bits.payload.data,
io.tl.acquire.bits.payload.data)
when(!active_out){
io.tl.release.ready := io.mem.req_data.ready
io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid
io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) ||
(io.tl.acquire.valid && acq_has_data)
when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) {
active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid
io.mem.req_cmd.valid := Bool(true)
cmd_sent_out := io.mem.req_cmd.ready
tag_out := io.mem.req_cmd.bits.tag
addr_out := io.mem.req_data.bits.data
has_data := io.mem.req_cmd.bits.rw
tl_done_out := tl_wrap_out
when(io.tl.release.valid) {
data_from_rel := Bool(true)
make_grant_ack := Bool(true)
io.mem.req_cmd.bits.rw := rel_has_data
io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id
io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block
io.mem.req_data.bits.data := io.tl.release.bits.payload.data
} .elsewhen(io.tl.acquire.valid) {
data_from_rel := Bool(false)
make_grant_ack := acq_has_data
io.mem.req_cmd.bits.rw := acq_has_data
io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id,
io.tl.acquire.bits.payload.a_type,
io.tl.acquire.bits.payload.is_builtin_type)
io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block
io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data
}
}
} }
when(tl_wrap_out) { tl_done_out := Bool(true) } when(active_out) {
when(mif_wrap_out) { mif_done_out := Bool(true) } io.mem.req_cmd.valid := !cmd_sent_out
when(cmd_sent_out && (!has_data || mif_done_out)) { cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire()
active_out := Bool(false) when(has_data && !tl_done_out) {
when(data_from_rel) {
io.tl.release.ready := io.mem.req_data.ready
io.mem.req_data.valid := io.tl.release.valid
} .otherwise {
io.tl.acquire.ready := io.mem.req_data.ready
io.mem.req_data.valid := io.tl.acquire.valid
}
}
when(tl_wrap_out) { tl_done_out := Bool(true) }
when(tl_done_out && make_grant_ack) {
gnt_arb.io.in(1).valid := Bool(true)
when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) }
}
when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) {
active_out := Bool(false)
}
} }
} }
mem_cmd_q.io.enq.bits.rw := has_data
mem_cmd_q.io.enq.bits.tag := tag_out
mem_cmd_q.io.enq.bits.addr := addr_out
mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data
io.mem.req_cmd <> mem_cmd_q.io.deq
io.mem.req_data <> mem_data_q.io.deq
// Aggregate incoming MemIO responses into TL Grants // Aggregate incoming MemIO responses into TL Grants
val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data
val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats)
val active_in = Reg(init=Bool(false)) val active_in = Reg(init=Bool(false))
val mif_done_in = Reg(init=Bool(false)) val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats)
val tag_in = Reg(UInt(width = mifTagBits)) val tag_in = Reg(UInt(width = mifTagBits))
val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) }
val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone }
tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits)
val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits)
val mif_prog_in = mif_cnt_in*UInt(mifDataBits)
when(!active_in) { if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) {
io.mem.resp.ready := Bool(true) val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data
when(io.mem.resp.valid) { val mif_done_in = Reg(init=Bool(false))
active_in := Bool(true) val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) }
mif_done_in := mif_wrap_in val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone }
tag_in := io.mem.resp.bits.tag tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits)
mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits)
} val mif_prog_in = mif_cnt_in*UInt(mifDataBits)
} gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = tag_in(0),
a_type = tag_in >> UInt(1),
client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = tl_buf_in(tl_cnt_in))
when(active_in) { when(!active_in) {
io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in
when(!mif_done_in) {
io.mem.resp.ready := Bool(true) io.mem.resp.ready := Bool(true)
when(io.mem.resp.valid) { when(io.mem.resp.valid) {
mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data active_in := Bool(true)
mif_done_in := mif_wrap_in
tag_in := io.mem.resp.bits.tag
mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data
} }
} }
when(mif_wrap_in) { mif_done_in := Bool(true) } when(active_in) {
when(tl_wrap_in) { active_in := Bool(false) } gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in
when(!mif_done_in) {
io.mem.resp.ready := Bool(true)
when(io.mem.resp.valid) {
mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data
}
}
when(mif_wrap_in) { mif_done_in := Bool(true) }
when(tl_wrap_in) { active_in := Bool(false) }
}
} else { // Don't generate all the uneeded data buffers and flow resp
gnt_arb.io.in(0).valid := io.mem.resp.valid
io.mem.resp.ready := gnt_arb.io.in(0).ready
gnt_arb.io.in(0).bits.payload :=
ManagerMetadata.onReset.makeGrant(
acq = Acquire(
is_builtin_type = io.mem.resp.bits.tag(0),
a_type = io.mem.resp.bits.tag >> UInt(1),
client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1),
addr_block = UInt(0)), //DNC
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = io.mem.resp.bits.data)
} }
io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true),
g_type = Grant.uncachedReadBlock,
client_xact_id = tag_in,
manager_xact_id = UInt(0),
addr_beat = tl_cnt_in,
data = tl_buf_in(tl_cnt_in))
} }
class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module
@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul
dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw
} }
class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module {
val io = new Bundle { val io = new Bundle {
val uncached = new UncachedTileLinkIO().flip val tl = new TileLinkIO().flip
val mem = new MemPipeIO val mem = new MemPipeIO
} }
val a = Module(new MemIOUncachedTileLinkIOConverter(2)) val a = Module(new MemIOTileLinkIOConverter(1))
val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles))
a.io.uncached <> io.uncached a.io.tl <> io.tl
b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true)
b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true)
a.io.mem.resp <> b.io.cpu.resp a.io.mem.resp <> b.io.cpu.resp

188
uncore/src/main/scala/metadata.scala Normal file
View File

@ -0,0 +1,188 @@
// See LICENSE for license details.
package uncore
import Chisel._
// Classes to represent coherence information in clients and managers
abstract class CoherenceMetadata extends Bundle {
val co = params(TLCoherencePolicy)
val id = params(TLId)
}
/* The ClientMetadata stores the client-side coherence information,
such as permissions on the data and whether the data is dirty.
Its API can be used to make TileLink messages in response to
memory operations or TileLink Probes.
*/
class ClientMetadata extends CoherenceMetadata {
val state = UInt(width = co.clientStateWidth)
def ===(rhs: ClientMetadata): Bool = this.state === rhs.state
def isValid(dummy: Int = 0): Bool = co.isValid(this)
def isHit(cmd: UInt): Bool = co.isHit(cmd, this)
def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this)
def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool =
co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this)
def requiresReleaseOnCacheControl(cmd: UInt): Bool =
co.requiresReleaseOnCacheControl(cmd: UInt, this)
def requiresVoluntaryWriteback(dummy: Int = 0): Bool =
co.requiresReleaseOnCacheControl(M_FLUSH, this)
def makeAcquire(
client_xact_id: UInt,
addr_block: UInt,
op_code: UInt): Acquire = {
Bundle(Acquire(
is_builtin_type = Bool(false),
a_type = co.getAcquireType(op_code, this),
client_xact_id = client_xact_id,
addr_block = addr_block,
union = Cat(op_code, Bool(true))),
{ case TLId => id })
}
def makeVoluntaryWriteback(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Bundle(Release(
voluntary = Bool(true),
r_type = co.getReleaseType(M_FLUSH, this),
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def makeRelease(
prb: Probe,
client_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Bundle(Release(
voluntary = Bool(false),
r_type = co.getReleaseType(prb, this),
client_xact_id = client_xact_id,
addr_block = prb.addr_block,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def onGrant(incoming: Grant, pending: UInt): ClientMetadata =
Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id })
def onProbe(incoming: Probe): ClientMetadata =
Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id })
def onHit(cmd: UInt): ClientMetadata =
Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id })
def onCacheControl(cmd: UInt): ClientMetadata =
Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id })
}
object ClientMetadata {
def apply(state: UInt) = {
val meta = new ClientMetadata
meta.state := state
meta
}
def onReset = new ClientMetadata().co.clientMetadataOnReset
}
/* The ManagerMetadata stores manager-side information about the status
of a cache block, including whether it has any known sharers. Its
API can be used to create Probe and Grant TileLink messages.
*/
class ManagerMetadata extends CoherenceMetadata {
// val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel
val sharers = UInt(width = co.dir.width)
def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel
this.sharers === rhs.sharers
def full(dummy: Int = 0) = co.dir.full(this.sharers)
def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this)
def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this)
def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool =
co.requiresProbes(M_FLUSH, this)
def makeProbe(cmd: UInt, addr_block: UInt): Probe =
Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id })
def makeProbe(acq: Acquire): Probe =
Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id })
def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe =
makeProbe(M_FLUSH, addr_block)
def makeGrant(rel: Release, manager_xact_id: UInt): Grant = {
Bundle(Grant(
is_builtin_type = Bool(true),
g_type = Grant.voluntaryAckType,
client_xact_id = rel.client_xact_id,
manager_xact_id = manager_xact_id), { case TLId => id })
}
def makeGrant(
acq: Acquire,
manager_xact_id: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = {
Bundle(Grant(
is_builtin_type = acq.isBuiltInType(),
g_type = Mux(acq.isBuiltInType(),
acq.getBuiltInGrantType(),
co.getGrantType(acq, this)),
client_xact_id = acq.client_xact_id,
manager_xact_id = manager_xact_id,
addr_beat = addr_beat,
data = data), { case TLId => id })
}
def onRelease(incoming: Release, src: UInt): ManagerMetadata =
Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id })
def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata =
Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id })
}
object ManagerMetadata {
def apply(sharers: UInt, state: UInt = UInt(width = 0)) = {
val meta = new ManagerMetadata
//meta.state := state TODO: Fix 0-width wires in Chisel
meta.sharers := sharers
meta
}
def apply() = {
val meta = new ManagerMetadata
//meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel
meta.sharers := meta.co.dir.flush
meta
}
def onReset = new ManagerMetadata().co.managerMetadataOnReset
}
/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy
that is a Manager with respect to some inner caches and a Client with
respect to some outer cache.
*/
class HierarchicalMetadata extends CoherenceMetadata {
val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)})
val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)})
def ===(rhs: HierarchicalMetadata): Bool =
this.inner === rhs.inner && this.outer === rhs.outer
def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs)
}
object HierarchicalMetadata {
def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = {
val m = new HierarchicalMetadata
m.inner := inner
m.outer := outer
m
}
def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset)
}
case object InnerTLId extends Field[String]
case object OuterTLId extends Field[String]

307
uncore/src/main/scala/tilelink.scala
View File

@ -8,30 +8,35 @@ import scala.math.max
// external requirements or design space exploration // external requirements or design space exploration
// //
case object TLId extends Field[String] // Unique name per network case object TLId extends Field[String] // Unique name per network
case object TLCoherence extends Field[CoherencePolicy] case object TLCoherencePolicy extends Field[CoherencePolicy]
case object TLAddrBits extends Field[Int] case object TLBlockAddrBits extends Field[Int]
case object TLManagerXactIdBits extends Field[Int] case object TLManagerXactIdBits extends Field[Int]
case object TLClientXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int]
case object TLDataBits extends Field[Int] case object TLDataBits extends Field[Int]
case object TLDataBeats extends Field[Int] case object TLDataBeats extends Field[Int]
case object TLNetworkIsOrderedP2P extends Field[Boolean]
abstract trait TileLinkParameters extends UsesParameters { abstract trait TileLinkParameters extends UsesParameters {
val tlBlockAddrBits = params(TLAddrBits) val tlBlockAddrBits = params(TLBlockAddrBits)
val tlClientXactIdBits = params(TLClientXactIdBits) val tlClientXactIdBits = params(TLClientXactIdBits)
val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits)
val tlDataBits = params(TLDataBits) val tlDataBits = params(TLDataBits)
val tlDataBeats = params(TLDataBeats) val tlDataBeats = params(TLDataBeats)
val tlCoh = params(TLCoherencePolicy)
val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8
val tlBeatAddrBits = log2Up(tlDataBeats) val tlBeatAddrBits = log2Up(tlDataBeats)
val tlByteAddrBits = log2Up(tlWriteMaskBits) val tlByteAddrBits = log2Up(tlWriteMaskBits)
val tlAtomicOpcodeBits = M_SZ val tlMemoryOpcodeBits = M_SZ
val tlUncachedOperandSizeBits = MT_SZ val tlMemoryOperandSizeBits = MT_SZ
val tlSubblockUnionBits = max(tlWriteMaskBits, val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes),
tlCoh.acquireTypeWidth)
val tlAcquireUnionBits = max(tlWriteMaskBits,
(tlByteAddrBits + (tlByteAddrBits +
tlUncachedOperandSizeBits + tlMemoryOperandSizeBits +
tlAtomicOpcodeBits)) + 1 tlMemoryOpcodeBits)) + 1
val co = params(TLCoherence) val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes),
val networkPreservesPointToPointOrdering = false //TODO: check physical network type tlCoh.grantTypeWidth) + 1
val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P)
} }
abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLBundle extends Bundle with TileLinkParameters
@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now
trait HasCacheBlockAddress extends TLBundle { trait HasCacheBlockAddress extends TLBundle {
val addr_block = UInt(width = tlBlockAddrBits) val addr_block = UInt(width = tlBlockAddrBits)
def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block
def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr def conflicts(addr: UInt) = this.addr_block === addr
} }
trait HasTileLinkBeatId extends TLBundle { trait HasTileLinkBeatId extends TLBundle {
@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle {
val manager_xact_id = Bits(width = tlManagerXactIdBits) val manager_xact_id = Bits(width = tlManagerXactIdBits)
} }
abstract trait HasTileLinkData extends HasTileLinkBeatId { trait HasTileLinkData extends HasTileLinkBeatId {
val data = UInt(width = tlDataBits) val data = UInt(width = tlDataBits)
def hasData(dummy: Int = 0): Bool def hasData(dummy: Int = 0): Bool
def hasMultibeatData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool
} }
@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel
with HasClientTransactionId with HasClientTransactionId
with HasTileLinkData { with HasTileLinkData {
// Actual bundle fields // Actual bundle fields
val builtin_type = Bool() val is_builtin_type = Bool()
val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val a_type = UInt(width = tlAcquireTypeBits)
val subblock = Bits(width = tlSubblockUnionBits) val union = Bits(width = tlAcquireUnionBits)
// Utility funcs for accessing subblock union // Utility funcs for accessing subblock union
val opSizeOff = tlByteAddrBits + 1 val opCodeOff = 1
val opCodeOff = tlUncachedOperandSizeBits + opSizeOff val opSizeOff = tlMemoryOpcodeBits + opCodeOff
val opMSB = tlAtomicOpcodeBits + opCodeOff val addrByteOff = tlMemoryOperandSizeBits + opSizeOff
def allocate(dummy: Int = 0) = subblock(0) val addrByteMSB = tlByteAddrBits + addrByteOff
def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) def allocate(dummy: Int = 0) = union(0)
def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff))
def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff)
def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff)
def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1)
def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte())
// Other helper funcs // Other helper funcs
def is(t: UInt) = a_type === t def is(t: UInt) = a_type === t
def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type)
// Assumes no custom types have data
def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type)
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() &&
Acquire.typesWithMultibeatData.contains(a_type) Acquire.typesWithMultibeatData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType
def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = def getBuiltInGrantType(dummy: Int = 0): UInt = {
Probe(co.getProbeType(this, meta), this.addr_block) MuxLookup(this.a_type, Grant.ackType, Array(
Acquire.getType -> Grant.dataBeatType,
def makeGrant( Acquire.getBlockType -> Grant.dataBlockType,
manager_xact_id: UInt, Acquire.putType -> Grant.ackType,
meta: ManagerMetadata = co.managerMetadataOnFlush, Acquire.putBlockType -> Grant.ackType,
addr_beat: UInt = UInt(0), Acquire.putAtomicType -> Grant.dataBeatType))
data: UInt = UInt(0)): Grant = {
Grant(
builtin_type = this.builtin_type,
g_type = co.getGrantType(this, meta),
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id,
addr_beat = addr_beat,
data = data
)
} }
} }
object Acquire { object Acquire {
val nBuiltinAcquireTypes = 5 val nBuiltInTypes = 5
//TODO: Use Enum //TODO: Use Enum
def uncachedRead = UInt(0) def getType = UInt("b000")
def uncachedReadBlock = UInt(1) def getBlockType = UInt("b001")
def uncachedWrite = UInt(2) def putType = UInt("b010")
def uncachedWriteBlock = UInt(3) def putBlockType = UInt("b011")
def uncachedAtomic = UInt(4) def putAtomicType = UInt("b100")
def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) def typesWithData = Vec(putType, putBlockType, putAtomicType)
def typesWithMultibeatData = Vec(uncachedWriteBlock) def typesWithMultibeatData = Vec(putBlockType)
def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock def typesOnSubBlocks = Vec(putType, getType, putAtomicType)
def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type)
//TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache:
def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock
def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt
// Most generic constructor // Most generic constructor
def apply( def apply(
builtin_type: Bool, is_builtin_type: Bool,
a_type: Bits, a_type: Bits,
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt, addr_block: UInt,
addr_beat: UInt = UInt(0), addr_beat: UInt = UInt(0),
data: UInt = UInt(0), data: UInt = UInt(0),
subblock: UInt = UInt(0)): Acquire = { union: UInt = UInt(0)): Acquire = {
val acq = new Acquire val acq = new Acquire
acq.builtin_type := builtin_type acq.is_builtin_type := is_builtin_type
acq.a_type := a_type acq.a_type := a_type
acq.client_xact_id := client_xact_id acq.client_xact_id := client_xact_id
acq.addr_block := addr_block acq.addr_block := addr_block
acq.addr_beat := addr_beat acq.addr_beat := addr_beat
acq.data := data acq.data := data
acq.subblock := subblock acq.union := union
acq acq
} }
// For cached types
def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = {
apply(
builtin_type = Bool(false),
a_type = a_type,
client_xact_id = client_xact_id,
addr_block = addr_block)
}
// Copy constructor // Copy constructor
def apply(a: Acquire): Acquire = { def apply(a: Acquire): Acquire = {
val acq = new Acquire val acq = new Acquire
@ -177,58 +169,73 @@ object Acquire {
} }
// Asks for a single TileLink beat of data // Asks for a single TileLink beat of data
object UncachedRead { object Get {
def apply( def apply(
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt, addr_block: UInt,
addr_beat: UInt, addr_beat: UInt,
alloc: Bool = Bool(true)): Acquire = { alloc: Bool = Bool(true)): Acquire = {
Acquire( Acquire(
builtin_type = Bool(true), is_builtin_type = Bool(true),
a_type = Acquire.uncachedRead, a_type = Acquire.getType,
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
addr_block = addr_block, addr_block = addr_block,
addr_beat = addr_beat, addr_beat = addr_beat,
subblock = alloc) union = Cat(M_XRD, alloc))
} }
} }
// Asks for an entire cache block of data // Asks for an entire cache block of data
object UncachedReadBlock { object GetBlock {
def apply( def apply(
client_xact_id: UInt = UInt(0), client_xact_id: UInt = UInt(0),
addr_block: UInt, addr_block: UInt,
alloc: Bool = Bool(true)): Acquire = { alloc: Bool = Bool(true)): Acquire = {
Acquire( Acquire(
builtin_type = Bool(true), is_builtin_type = Bool(true),
a_type = Acquire.uncachedReadBlock, a_type = Acquire.getBlockType,
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
addr_block = addr_block, addr_block = addr_block,
subblock = alloc.toUInt) union = Cat(M_XRD, alloc))
} }
} }
object UncachedWrite { // Writes up to a single TileLink beat of data, using mask
object Put {
def apply( def apply(
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt, addr_block: UInt,
addr_beat: UInt, addr_beat: UInt,
data: UInt, data: UInt,
write_mask: UInt = Acquire.fullWriteMask, write_mask: UInt = Acquire.fullWriteMask): Acquire = {
alloc: Bool = Bool(true)): Acquire = {
Acquire( Acquire(
builtin_type = Bool(true), is_builtin_type = Bool(true),
a_type = Acquire.uncachedWrite, a_type = Acquire.putType,
addr_block = addr_block, addr_block = addr_block,
addr_beat = addr_beat, addr_beat = addr_beat,
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
data = data, data = data,
subblock = Cat(write_mask, alloc)) union = Cat(write_mask, Bool(true)))
} }
} }
// For full block of data // Writes an entire cache block of data
object UncachedWriteBlock { object PutBlock {
def apply(
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt,
data: UInt,
write_mask: UInt): Acquire = {
Acquire(
is_builtin_type = Bool(true),
a_type = Acquire.putBlockType,
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data,
union = Cat(write_mask, (write_mask != Acquire.fullWriteMask)))
}
def apply( def apply(
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt, addr_block: UInt,
@ -236,17 +243,18 @@ object UncachedWriteBlock {
data: UInt, data: UInt,
alloc: Bool = Bool(true)): Acquire = { alloc: Bool = Bool(true)): Acquire = {
Acquire( Acquire(
builtin_type = Bool(true), is_builtin_type = Bool(true),
a_type = Acquire.uncachedWriteBlock, a_type = Acquire.putBlockType,
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
addr_block = addr_block, addr_block = addr_block,
addr_beat = addr_beat, addr_beat = addr_beat,
data = data, data = data,
subblock = Cat(Acquire.fullWriteMask, alloc)) union = Cat(Acquire.fullWriteMask, alloc))
} }
} }
object UncachedAtomic { // Performs an atomic operation in the outer memory
object PutAtomic {
def apply( def apply(
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt, addr_block: UInt,
@ -256,48 +264,30 @@ object UncachedAtomic {
operand_size: UInt, operand_size: UInt,
data: UInt): Acquire = { data: UInt): Acquire = {
Acquire( Acquire(
builtin_type = Bool(true), is_builtin_type = Bool(true),
a_type = Acquire.uncachedAtomic, a_type = Acquire.putAtomicType,
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
addr_block = addr_block, addr_block = addr_block,
addr_beat = addr_beat, addr_beat = addr_beat,
data = data, data = data,
subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true)))
} }
} }
class Probe extends ManagerToClientChannel class Probe extends ManagerToClientChannel
with HasCacheBlockAddress { with HasCacheBlockAddress {
val p_type = UInt(width = co.probeTypeWidth) val p_type = UInt(width = tlCoh.probeTypeWidth)
def is(t: UInt) = p_type === t def is(t: UInt) = p_type === t
def makeRelease(
client_xact_id: UInt,
meta: ClientMetadata = co.clientMetadataOnFlush,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(false),
r_type = co.getReleaseType(this, meta),
client_xact_id = client_xact_id,
addr_block = this.addr_block,
addr_beat = addr_beat,
data = data)
}
} }
object Probe { object Probe {
val co = new Probe().co
def apply(p_type: UInt, addr_block: UInt) = { def apply(p_type: UInt, addr_block: UInt) = {
val prb = new Probe val prb = new Probe
prb.p_type := p_type prb.p_type := p_type
prb.addr_block := addr_block prb.addr_block := addr_block
prb prb
} }
def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = {
apply(co.getProbeType(M_FLUSH, meta), addr_block)
}
} }
@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel
with HasCacheBlockAddress with HasCacheBlockAddress
with HasClientTransactionId with HasClientTransactionId
with HasTileLinkData { with HasTileLinkData {
val r_type = UInt(width = co.releaseTypeWidth) val r_type = UInt(width = tlCoh.releaseTypeWidth)
val voluntary = Bool() val voluntary = Bool()
// Helper funcs // Helper funcs
def is(t: UInt) = r_type === t def is(t: UInt) = r_type === t
def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type)
//TODO: Assumes all releases write back full cache blocks: //TODO: Assumes all releases write back full cache blocks:
def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type)
def isVoluntary(dummy: Int = 0) = voluntary def isVoluntary(dummy: Int = 0) = voluntary
def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering)
def makeGrant(
manager_xact_id: UInt,
meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = {
Grant(
g_type = Grant.voluntaryAck,
builtin_type = Bool(true), // Grant.voluntaryAck is built-in type
client_xact_id = this.client_xact_id,
manager_xact_id = manager_xact_id
)
}
} }
object Release { object Release {
val co = new Release().co
def apply( def apply(
voluntary: Bool, voluntary: Bool,
r_type: UInt, r_type: UInt,
@ -346,68 +324,52 @@ object Release {
rel.voluntary := voluntary rel.voluntary := voluntary
rel rel
} }
def makeVoluntaryWriteback(
meta: ClientMetadata,
client_xact_id: UInt,
addr_block: UInt,
addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Release = {
Release(
voluntary = Bool(true),
r_type = co.getReleaseType(M_FLUSH, meta),
client_xact_id = client_xact_id,
addr_block = addr_block,
addr_beat = addr_beat,
data = data)
}
} }
class Grant extends ManagerToClientChannel class Grant extends ManagerToClientChannel
with HasTileLinkData with HasTileLinkData
with HasClientTransactionId with HasClientTransactionId
with HasManagerTransactionId { with HasManagerTransactionId {
val builtin_type = Bool() val is_builtin_type = Bool()
val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) val g_type = UInt(width = tlGrantTypeBits)
// Helper funcs // Helper funcs
def is(t: UInt) = g_type === t def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type
def hasData(dummy: Int = 0): Bool = Mux(builtin_type, def is(t: UInt):Bool = g_type === t
def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(),
Grant.typesWithData.contains(g_type), Grant.typesWithData.contains(g_type),
co.grantTypesWithData.contains(g_type)) tlCoh.grantTypesWithData.contains(g_type))
def hasMultibeatData(dummy: Int = 0): Bool = def hasMultibeatData(dummy: Int = 0): Bool =
Bool(tlDataBeats > 1) && Mux(builtin_type, Bool(tlDataBeats > 1) && Mux(isBuiltInType(),
Grant.typesWithMultibeatData.contains(g_type), Grant.typesWithMultibeatData.contains(g_type),
co.grantTypesWithData.contains(g_type)) tlCoh.grantTypesWithData.contains(g_type))
def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType)
def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary()
def makeFinish(dummy: Int = 0): Finish = { def makeFinish(dummy: Int = 0): Finish = {
val f = new Finish val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits })
f.manager_xact_id := this.manager_xact_id f.manager_xact_id := this.manager_xact_id
f f
} }
} }
object Grant { object Grant {
val nBuiltinGrantTypes = 5 val nBuiltInTypes = 4
//TODO Use Enum def voluntaryAckType = UInt("b00")
def voluntaryAck = UInt(0) def ackType = UInt("b01")
def uncachedRead = UInt(1) def dataBeatType = UInt("b10")
def uncachedReadBlock = UInt(2) def dataBlockType = UInt("b11")
def uncachedWrite = UInt(3) def typesWithData = Vec(dataBlockType, dataBeatType)
def uncachedAtomic = UInt(4) def typesWithMultibeatData= Vec(dataBlockType)
def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic)
def typesWithMultibeatData= Vec(uncachedReadBlock)
def apply( def apply(
builtin_type: Bool, is_builtin_type: Bool,
g_type: UInt, g_type: UInt,
client_xact_id: UInt, client_xact_id: UInt,
manager_xact_id: UInt, manager_xact_id: UInt,
addr_beat: UInt = UInt(0), addr_beat: UInt = UInt(0),
data: UInt = UInt(0)): Grant = { data: UInt = UInt(0)): Grant = {
val gnt = new Grant val gnt = new Grant
gnt.builtin_type := builtin_type gnt.is_builtin_type := is_builtin_type
gnt.g_type := g_type gnt.g_type := g_type
gnt.client_xact_id := client_xact_id gnt.client_xact_id := client_xact_id
gnt.manager_xact_id := manager_xact_id gnt.manager_xact_id := manager_xact_id
@ -415,23 +377,12 @@ object Grant {
gnt.data := data gnt.data := data
gnt gnt
} }
def getGrantTypeForUncached(a: Acquire): UInt = {
MuxLookup(a.a_type, Grant.uncachedRead, Array(
Acquire.uncachedRead -> Grant.uncachedRead,
Acquire.uncachedReadBlock -> Grant.uncachedReadBlock,
Acquire.uncachedWrite -> Grant.uncachedWrite,
Acquire.uncachedWriteBlock -> Grant.uncachedWrite,
Acquire.uncachedAtomic -> Grant.uncachedAtomic
))
}
} }
class Finish extends ClientToManagerChannel with HasManagerTransactionId class Finish extends ClientToManagerChannel with HasManagerTransactionId
// Complete IO definitions for two types of TileLink clients // Complete IO definitions for two types of TileLink clients
class UncachedTileLinkIO extends Bundle { class UncachedTileLinkIO extends TLBundle {
val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire))
val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip
val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) val finish = new DecoupledIO(new LogicalNetworkIO(new Finish))
@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule {
io.out.release.valid := Bool(false) io.out.release.valid := Bool(false)
} }
object TileLinkIOWrapper { object TileLinkIOWrapper {
def apply[T <: Data](utl: UncachedTileLinkIO) = { def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = {
val conv = Module(new TileLinkIOWrapper)(p)
conv.io.in <> utl
conv.io.out
}
def apply(utl: UncachedTileLinkIO): TileLinkIO = {
val conv = Module(new TileLinkIOWrapper) val conv = Module(new TileLinkIOWrapper)
conv.io.in <> utl conv.io.in <> utl
conv.io.out conv.io.out
} }
def apply(tl: TileLinkIO) = tl
} }
abstract trait HasArbiterTypes { abstract trait HasArbiterTypes {

508
uncore/src/main/scala/uncore.scala
View File

@ -2,437 +2,147 @@
package uncore package uncore
import Chisel._ import Chisel._
import scala.reflect.ClassTag
case object NReleaseTransactors extends Field[Int] case object NReleaseTransactors extends Field[Int]
case object NProbeTransactors extends Field[Int] case object NProbeTransactors extends Field[Int]
case object NAcquireTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int]
case object NIncoherentClients extends Field[Int] case object NIncoherentClients extends Field[Int]
case object NCoherentClients extends Field[Int] case object NCoherentClients extends Field[Int]
case object L2StoreDataQueueDepth extends Field[Int] case object L2CoherencePolicy extends Field[CoherencePolicy]
case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy]
case object L2DirectoryRepresentation extends Field[DirectoryRepresentation]
abstract trait CoherenceAgentParameters extends UsesParameters trait CoherenceAgentParameters extends UsesParameters {
with TileLinkParameters {
val nReleaseTransactors = 1 val nReleaseTransactors = 1
val nAcquireTransactors = params(NAcquireTransactors) val nAcquireTransactors = params(NAcquireTransactors)
val nTransactors = nReleaseTransactors + nAcquireTransactors val nTransactors = nReleaseTransactors + nAcquireTransactors
val nCoherentClients = params(NCoherentClients) val nCoherentClients = params(NCoherentClients)
val nIncoherentClients = params(NIncoherentClients) val nIncoherentClients = params(NIncoherentClients)
val nClients = nCoherentClients + nIncoherentClients val nClients = nCoherentClients + nIncoherentClients
val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)})
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val outerDataBeats = outerTLParams(TLDataBeats)
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases val outerDataBits = outerTLParams(TLDataBits)
def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)})
val innerDataBeats = innerTLParams(TLDataBeats)
val innerDataBits = innerTLParams(TLDataBits)
val innerBeatAddrBits = log2Up(innerDataBeats)
val innerByteAddrBits = log2Up(innerDataBits/8)
require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement
} }
abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters
abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters
class DataQueueLocation extends Bundle with CoherenceAgentParameters { trait HasCoherenceAgentWiringHelpers {
val idx = UInt(width = dqIdxBits) def doOutputArbitration[T <: Data : ClassTag](
val loc = UInt(width = log2Ceil(nDataQueueLocations)) out: DecoupledIO[T],
} ins: Seq[DecoupledIO[T]]) {
object DataQueueLocation { val arb = Module(new RRArbiter(out.bits.clone, ins.size))
def apply(idx: UInt, loc: UInt) = { out <> arb.io.out
val d = new DataQueueLocation arb.io.in zip ins map { case (a, in) => a <> in }
d.idx := idx }
d.loc := loc
d def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag](
out: DecoupledIO[S],
ins: Seq[DecoupledIO[S]]) {
def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData()
val arb = Module(new LockingRRArbiter(
out.bits.clone,
ins.size,
out.bits.payload.tlDataBeats,
lock _))
out <> arb.io.out
arb.io.in zip ins map { case (a, in) => a <> in }
}
def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) {
val idx = in.bits.id
outs.map(_.bits := in.bits)
outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
}
def doInputRouting[T <: HasManagerTransactionId](
in: DecoupledIO[LogicalNetworkIO[T]],
outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) {
val idx = in.bits.payload.manager_xact_id
outs.map(_.bits := in.bits)
outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
in.ready := Vec(outs.map(_.ready)).read(idx)
} }
} }
abstract class CoherenceAgent(innerId: String, outerId: String) extends Module trait HasInnerTLIO extends CoherenceAgentBundle {
with CoherenceAgentParameters { val inner = Bundle(new TileLinkIO)(innerTLParams).flip
val io = new Bundle { val incoherent = Vec.fill(nClients){Bool()}.asInput
val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip def iacq(dummy: Int = 0) = inner.acquire.bits.payload
val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) def iprb(dummy: Int = 0) = inner.probe.bits.payload
val incoherent = Vec.fill(nClients){Bool()}.asInput def irel(dummy: Int = 0) = inner.release.bits.payload
} def ignt(dummy: Int = 0) = inner.grant.bits.payload
def ifin(dummy: Int = 0) = inner.finish.bits.payload
} }
class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends trait HasUncachedOuterTLIO extends CoherenceAgentBundle {
CoherenceAgent(innerId, outerId) { val outer = Bundle(new UncachedTileLinkIO)(outerTLParams)
def oacq(dummy: Int = 0) = outer.acquire.bits.payload
val internalDataBits = new DataQueueLocation().getWidth def ognt(dummy: Int = 0) = outer.grant.bits.payload
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) def ofin(dummy: Int = 0) = outer.finish.bits.payload
// Create SHRs for outstanding transactions
val trackerList = (0 until nReleaseTransactors).map(id =>
Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++
(nReleaseTransactors until nTransactors).map(id =>
Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits}))
// Propagate incoherence flags
trackerList.map(_.io.tile_incoherent := io.incoherent.toBits)
// Queue to store impending UncachedWrite data
val acquire = io.inner.acquire
val sdq_val = Reg(init=Bits(0, sdqDepth))
val sdq_alloc_id = PriorityEncoder(~sdq_val)
val sdq_rdy = !sdq_val.andR
val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
// Handle acquire transaction initiation
val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
val block_acquires = any_acquire_conflict
val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
alloc_arb.io.in(i).valid := t.acquire.ready
t.acquire.bits := acquire.bits
t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
t.acquire.valid := alloc_arb.io.in(i).ready
}
acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
// Queue to store impending Voluntary Release data
val release = io.inner.release
val voluntary = release.bits.payload.isVoluntary()
val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width
val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1
when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
// Handle releases, which might be voluntary and might have data
val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b}
for( i <- 0 until trackerList.size ) {
val t = trackerList(i).io.inner
t.release.bits := release.bits
t.release.bits.payload.data := (if (i < nReleaseTransactors)
DataQueueLocation(rel_data_cnt, inVolWBQueue)
else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits
t.release.valid := release.valid && (release_idx === UInt(i))
}
release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx)
// Wire finished transaction acks
val ack = io.inner.finish
trackerList.map(_.io.inner.finish.valid := ack.valid)
trackerList.map(_.io.inner.finish.bits := ack.bits)
ack.ready := Bool(true)
// Wire probe requests to clients
val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size))
io.inner.probe <> probe_arb.io.out
probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe }
// Wire grant reply to initiating client
def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData()
val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _)))
io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat
io.inner.grant <> grant_arb.io.out
grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant }
// Create an arbiter for the one memory port
val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
{ case TLId => outerId; case TLDataBits => internalDataBits })
outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer }
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.payload.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
io.outer <> outer_arb.io.out
// Update SDQ valid bits
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
} }
trait HasCachedOuterTLIO extends CoherenceAgentBundle {
abstract class XactTracker(innerId: String, outerId: String) extends Module { val outer = Bundle(new TileLinkIO)(outerTLParams)
val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) def oacq(dummy: Int = 0) = outer.acquire.bits.payload
val nClients = params(NCoherentClients) + params(NIncoherentClients) def oprb(dummy: Int = 0) = outer.probe.bits.payload
val io = new Bundle { def orel(dummy: Int = 0) = outer.release.bits.payload
val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip def ognt(dummy: Int = 0) = outer.grant.bits.payload
val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) def ofin(dummy: Int = 0) = outer.finish.bits.payload
val tile_incoherent = Bits(INPUT, nClients)
val has_acquire_conflict = Bool(OUTPUT)
val has_release_match = Bool(OUTPUT)
}
val cacq = io.inner.acquire.bits
val crel = io.inner.release.bits
val cgnt = io.inner.grant.bits
val cfin = io.inner.finish.bits
val macq = io.outer.acquire.bits
val mgnt = io.outer.grant.bits
} }
class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO
val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
val state = Reg(init=s_idle)
val xact_src = Reg(io.inner.release.bits.header.src.clone) abstract class CoherenceAgent extends CoherenceAgentModule {
val xact_r_type = Reg(io.inner.release.bits.payload.r_type) def innerTL: TileLinkIO
val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) def outerTL: TileLinkIO
val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) def incoherent: Vec[Bool]
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
val xact = Release(
voluntary = Bool(true),
r_type = xact_r_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block)
val collect_inner_data = Reg(init=Bool(false))
// TODO: assert that all releases have full blocks of data
val (inner_data_cnt, inner_data_done) =
Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats)
val (outer_data_cnt, outer_data_done) =
Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats)
io.has_acquire_conflict := Bool(false)
io.has_release_match := crel.payload.isVoluntary()
io.outer.grant.ready := Bool(false)
io.outer.acquire.valid := Bool(false)
io.inner.acquire.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.grant.valid := Bool(false)
io.inner.finish.ready := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId))
io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
when(collect_inner_data) {
io.inner.release.ready := Bool(true)
when(io.inner.release.valid) {
xact_data(inner_data_cnt) := crel.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
}
switch (state) {
is(s_idle) {
io.inner.release.ready := Bool(true)
when( io.inner.release.valid ) {
xact_src := crel.header.src
xact_r_type := crel.payload.r_type
xact_addr_block := crel.payload.addr_block
xact_client_xact_id := crel.payload.client_xact_id
xact_data(UInt(0)) := crel.payload.data
collect_inner_data := crel.payload.hasMultibeatData()
state := Mux(crel.payload.hasData(), s_outer,
Mux(crel.payload.requiresAck(), s_ack, s_idle))
}
}
is(s_outer) {
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
when(outer_data_done) {
state := Mux(xact.requiresAck(), s_grant, s_idle)
}
}
is(s_grant) {
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) {
// TODO: This state is unnecessary if no client will ever issue the
// pending Acquire that caused this writeback until it receives the
// Grant.voluntaryAck for this writeback
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
} }
class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { abstract class ManagerCoherenceAgent extends CoherenceAgent
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) with HasCoherenceAgentWiringHelpers {
val state = Reg(init=s_idle) val io = new ManagerTLIO
def innerTL = io.inner
def outerTL = TileLinkIOWrapper(io.outer, outerTLParams)
def incoherent = io.incoherent
}
val xact_src = Reg(io.inner.acquire.bits.header.src.clone) class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO
val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone)
val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone)
val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone)
val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
val xact = Acquire(
builtin_type = xact_builtin_type,
a_type = xact_a_type,
client_xact_id = xact_client_xact_id,
addr_block = xact_addr_block,
addr_beat = xact_addr_beat,
data = UInt(0),
subblock = xact_subblock)
val collect_inner_data = Reg(init=Bool(false)) abstract class HierarchicalCoherenceAgent extends CoherenceAgent {
//TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock val io = new HierarchicalTLIO
val (inner_data_cnt, inner_data_done) = def innerTL = io.inner
Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) def outerTL = io.outer
val (outer_data_cnt, outer_data_done) = def incoherent = io.incoherent
Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) }
val (cgnt_data_cnt, cgnt_data_done) =
Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats)
trait HasTrackerConflictIO extends Bundle {
val has_acquire_conflict = Bool(OUTPUT)
val has_acquire_match = Bool(OUTPUT)
val has_release_match = Bool(OUTPUT)
}
val release_count = Reg(init=UInt(0, width = log2Up(nClients))) class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO
val probe_flags = Reg(init=Bits(0, width = nClients)) class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO
val curr_p_id = PriorityEncoder(probe_flags)
val pending_outer_write = xact.hasData() abstract class XactTracker extends CoherenceAgentModule {
val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = {
val multi = data.hasMultibeatData()
val probe_initial_flags = Bits(width = nClients) val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats)
probe_initial_flags := Bits(0) val cnt = Mux(multi, multi_cnt, beat)
// issue self-probes for uncached read xacts to facilitate I$ coherence val done = Mux(multi, multi_done, inc)
val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() (cnt, done)
val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0)))
probe_initial_flags := ~(io.tile_incoherent | myflag)
io.has_acquire_conflict := xact.conflicts(cacq.payload) &&
!collect_inner_data &&
(state != s_idle)
io.has_release_match := xact.conflicts(crel.payload) &&
!crel.payload.isVoluntary() &&
(state != s_idle)
val outer_write_acq = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = outer_data_cnt,
data = xact_data(outer_data_cnt)),
{ case TLId => outerId })
val outer_write_rel = Bundle(UncachedWriteBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block,
addr_beat = crel.payload.addr_beat,
data = crel.payload.data),
{ case TLId => outerId })
val outer_read = Bundle(UncachedReadBlock(
client_xact_id = UInt(trackerId),
addr_block = xact_addr_block),
{ case TLId => outerId })
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits.payload := outer_read //default
io.outer.grant.ready := io.inner.grant.ready
io.inner.probe.valid := Bool(false)
io.inner.probe.bits.header.src := UInt(bankId)
io.inner.probe.bits.header.dst := curr_p_id
io.inner.probe.bits.payload := xact.makeProbe()
io.inner.grant.valid := Bool(false)
io.inner.grant.bits.header.src := UInt(bankId)
io.inner.grant.bits.header.dst := xact_src
io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
when(collect_inner_data) {
io.inner.acquire.ready := Bool(true)
when(io.inner.acquire.valid) {
xact_data(inner_data_cnt) := cacq.payload.data
}
when(inner_data_done) { collect_inner_data := Bool(false) }
} }
def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](
switch (state) { in: DecoupledIO[LogicalNetworkIO[T]],
is(s_idle) { beat: UInt = UInt(0)) = {
io.inner.acquire.ready := Bool(true) connectDataBeatCounter(in.fire(), in.bits.payload, beat)
val needs_outer_write = cacq.payload.hasData() }
val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = {
when(io.inner.acquire.valid) { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2
xact_builtin_type := cacq.payload.builtin_type
xact_a_type := cacq.payload.a_type
xact_addr_block := cacq.payload.addr_block
xact_addr_beat := cacq.payload.addr_beat
xact_client_xact_id := cacq.payload.client_xact_id
xact_data(UInt(0)) := cacq.payload.data
xact_subblock := cacq.payload.subblock
xact_src := cacq.header.src
collect_inner_data := cacq.payload.hasMultibeatData()
probe_flags := probe_initial_flags
release_count := PopCount(probe_initial_flags)
state := Mux(probe_initial_flags.orR, s_probe,
Mux(needs_outer_write, s_mem_write,
Mux(needs_outer_read, s_mem_read, s_make_grant)))
}
}
is(s_probe) {
// Generate probes
io.inner.probe.valid := probe_flags.orR
when(io.inner.probe.ready) {
probe_flags := probe_flags & ~(UIntToOH(curr_p_id))
}
// Handle releases, which may have data to be written back
io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready
when(io.inner.release.valid) {
when(crel.payload.hasData()) {
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_write_rel
when(io.outer.acquire.ready) {
when(outer_data_done) {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
} .otherwise {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
}
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := Bool(true)
io.outer.acquire.bits.payload := outer_read
when(io.outer.acquire.ready) { state := s_mem_resp }
}
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true)
else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
io.outer.acquire.bits.payload := outer_write_acq
when(outer_data_done) {
state := Mux(pending_outer_read, s_mem_read, s_make_grant)
}
}
is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_mem_resp) { // Wait to forward grants from outer memory
when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) {
io.inner.grant.valid := Bool(true)
}
when(cgnt_data_done) {
state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) {
state := s_idle
}
}
} }
} }