1
0

Refactor the TransactionTracker logic in all the L2 TileLink Managers.

They now share common sub-transactions within traits, and use a common
set of state transitions and scoreboarding logic. Tracker allocation
logic has also been updated. No changes to external IOs or the TileLink protocol.
A new bufferless Broadcast hub is also included, but does not yet pass fuzzing checks.
This commit is contained in:
Henry Cook 2016-04-04 22:17:11 -07:00 committed by Howard Mao
parent 2d2096e509
commit 16bfbda3c9
9 changed files with 1486 additions and 1114 deletions

View File

@ -4,390 +4,198 @@ package uncore
import Chisel._ import Chisel._
import cde.{Parameters, Field} import cde.{Parameters, Field}
case object L2StoreDataQueueDepth extends Field[Int] class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
trait HasBroadcastHubParameters extends HasCoherenceAgentParameters { // Create TSHRs for outstanding transactions
val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats val irelTrackerList =
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
}
class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p)
with HasBroadcastHubParameters {
val idx = UInt(width = dqIdxBits)
val loc = UInt(width = log2Ceil(nDataQueueLocations))
}
object DataQueueLocation {
def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = {
val d = Wire(new DataQueueLocation)
d.idx := idx
d.loc := loc
d
}
}
class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p)
with HasBroadcastHubParameters {
val internalDataBits = new DataQueueLocation().getWidth
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
val usingStoreDataQueue = p.alterPartial({
case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
})
// Create SHRs for outstanding transactions
val trackerList =
(0 until nReleaseTransactors).map(id => (0 until nReleaseTransactors).map(id =>
Module(new BroadcastVoluntaryReleaseTracker(id)(usingStoreDataQueue))) ++ Module(new BufferedBroadcastVoluntaryReleaseTracker(id)))
val iacqTrackerList =
(nReleaseTransactors until nTransactors).map(id => (nReleaseTransactors until nTransactors).map(id =>
Module(new BroadcastAcquireTracker(id)(usingStoreDataQueue))) Module(new BufferedBroadcastAcquireTracker(id)))
val trackerList = irelTrackerList ++ iacqTrackerList
// Propagate incoherence flags // Propagate incoherence flags
trackerList.map(_.io.incoherent := io.incoherent) trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
// Queue to store impending Put data // Create an arbiter for the one memory port
val sdq = Reg(Vec(sdqDepth, io.iacq().data)) val outerList = trackerList.map(_.io.outer)
val sdq_val = Reg(init=Bits(0, sdqDepth)) val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
val sdq_alloc_id = PriorityEncoder(~sdq_val) (p.alterPartial({ case TLId => p(OuterTLId) })))
val sdq_rdy = !sdq_val.andR outer_arb.io.in <> outerList
val sdq_enq = trackerList.map( t => io.outer <> outer_arb.io.out
(t.io.alloc.iacq || t.io.matches.iacq) &&
t.io.inner.acquire.fire() &&
t.io.iacq().hasData()
).reduce(_||_)
when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data }
// Handle acquire transaction initiation // Handle acquire transaction initiation
val irel_vs_iacq_conflict = val irel_vs_iacq_conflict =
io.inner.acquire.valid && io.inner.acquire.valid &&
io.inner.release.valid && io.inner.release.valid &&
io.irel().conflicts(io.iacq()) io.irel().conflicts(io.iacq())
val sdqLoc = List.fill(nTransactors) {
DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
}
doInputRoutingWithAllocation(
io.inner.acquire,
trackerList.map(_.io.inner.acquire),
trackerList.map(_.io.matches.iacq),
trackerList.map(_.io.alloc.iacq),
Some(sdqLoc),
Some(sdq_rdy && !irel_vs_iacq_conflict),
Some(sdq_rdy))
// Queue to store impending Voluntary Release data doInputRoutingWithAllocation(
val voluntary = io.irel().isVoluntary() in = io.inner.acquire,
val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() outs = trackerList.map(_.io.inner.acquire),
val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width allocs = trackerList.map(_.io.alloc_iacq),
val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1 allocOverride = !irel_vs_iacq_conflict)
when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data }
// Handle releases, which might be voluntary and might have data // Handle releases, which might be voluntary and might have data
val vwbqLoc = (0 until nTransactors).map(i =>
(DataQueueLocation(rel_data_cnt,
(if(i < nReleaseTransactors) inVolWBQueue
else inClientReleaseQueue)).toBits))
doInputRoutingWithAllocation( doInputRoutingWithAllocation(
io.inner.release, in = io.inner.release,
trackerList.map(_.io.inner.release), outs = trackerList.map(_.io.inner.release),
trackerList.map(_.io.matches.irel), allocs = trackerList.map(_.io.alloc_irel))
trackerList.map(_.io.alloc.irel),
Some(vwbqLoc))
// Wire probe requests and grant reply to clients, finish acks from clients // Wire probe requests and grant reply to clients, finish acks from clients
// Note that we bypass the Grant data subbundles
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
io.inner.grant.bits.data := io.outer.grant.bits.data
io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
// Create an arbiter for the one memory port
val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size)
(usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) })))
outer_arb.io.in <> trackerList.map(_.io.outer)
// Get the pending data out of the store data queue
val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data)
val is_in_sdq = outer_data_ptr.loc === inStoreQueue
val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.hasData() &&
outer_data_ptr.loc === inStoreQueue
io.outer <> outer_arb.io.out
io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
// Update SDQ valid bits
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
} }
class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) {
val io = new ManagerXactTrackerIO val io = new HierarchicalXactTrackerIO
pinAllReadyValidLow(io) pinAllReadyValidLow(io)
} }
class BroadcastVoluntaryReleaseTracker(trackerId: Int) trait BroadcastsToAllClients extends HasCoherenceAgentParameters {
(implicit p: Parameters) extends BroadcastXactTracker()(p) { val coh = HierarchicalMetadata.onReset
val s_idle :: s_busy :: Nil = Enum(UInt(), 2) val inner_coh = coh.inner
val state = Reg(init=s_idle) val outer_coh = coh.outer
def full_representation = ~UInt(0, width = innerNCachingClients)
val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId })))
val coh = ManagerMetadata.onReset
val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats))
val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats))
val pending_ignt = Reg(init=Bool(false))
val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt)
// Accept a voluntary Release (and any further beats of data)
pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release))
io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR
when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data }
// Write the voluntarily written back data to outer memory using an Acquire.PutBlock
//TODO: Use io.outer.release instead?
pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) |
addPendingBitWhenBeatHasData(io.inner.release)
val curr_write_beat = PriorityEncoder(pending_writes)
io.outer.acquire.valid := state === s_busy && pending_writes.orR
io.outer.acquire.bits := PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = curr_write_beat,
data = xact.data_buffer(curr_write_beat))
(p.alterPartial({ case TLId => outerTLId }))
// Send an acknowledgement
io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid
io.inner.grant.bits := coh.makeGrant(xact)
when(io.inner.grant.fire()) { pending_ignt := Bool(false) }
io.outer.grant.ready := state === s_busy && io.inner.grant.ready
// State machine updates and transaction handler metadata intialization
when(state === s_idle && io.inner.release.valid && io.alloc.irel) {
xact := io.irel()
when(io.irel().hasMultibeatData()) {
pending_irels := dropPendingBitWhenBeatHasData(io.inner.release)
}. otherwise {
pending_irels := UInt(0)
} }
pending_writes := addPendingBitWhenBeatHasData(io.inner.release)
pending_ignt := io.irel().requiresAck()
state := s_busy
}
when(state === s_busy && all_pending_done) { state := s_idle }
// These IOs are used for routing in the parent abstract class BroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) extends VoluntaryReleaseTracker(trackerId)(p)
io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && io.irel().isVoluntary() with EmitsVoluntaryReleases
io.matches.oprb := Bool(false) with BroadcastsToAllClients {
val io = new HierarchicalXactTrackerIO
pinAllReadyValidLow(io)
// Checks for illegal behavior // Checks for illegal behavior
assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), assert(!(state === s_idle && io.inner.release.fire() && io.alloc_irel.should && !io.irel().isVoluntary()),
"VoluntaryReleaseTracker accepted Release that wasn't voluntary!") "VoluntaryReleaseTracker accepted Release that wasn't voluntary!")
} }
class BroadcastAcquireTracker(trackerId: Int) abstract class BroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
(implicit p: Parameters) extends BroadcastXactTracker()(p) { extends AcquireTracker(trackerId)(p)
val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) with EmitsVoluntaryReleases
val state = Reg(init=s_idle) with BroadcastsToAllClients {
val io = new HierarchicalXactTrackerIO
pinAllReadyValidLow(io)
val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val alwaysWriteFullBeat = false
val coh = ManagerMetadata.onReset val nSecondaryMisses = 1
def iacq_can_merge = Bool(false)
assert(!(state =/= s_idle && xact.isBuiltInType() && // Checks for illegal behavior
Vec(Acquire.putAtomicType, Acquire.getPrefetchType, Acquire.putPrefetchType).contains(xact.a_type)), // TODO: this could be allowed, but is a useful check against allocation gone wild
"Broadcast Hub does not support PutAtomics or prefetches") // TODO assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc_iacq.should &&
io.iacq().hasMultibeatData() && !io.iacq().first()),
val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1)))
val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients))
val curr_p_id = PriorityEncoder(pending_probes)
val mask_self = SInt(-1, width = io.inner.tlNCachingClients)
.toUInt
.bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe())
val mask_incoherent = mask_self & ~io.incoherent.toBits
val collect_iacq_data = Reg(init=Bool(false))
val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats))
val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire)
val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant)
val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
val pending_ognt_ack = Reg(init=Bool(false))
val pending_outer_write = xact.hasData()
val pending_outer_write_ = io.iacq().hasData()
val pending_outer_read = io.ignt().hasData()
val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData()
val subblock_type = xact.isSubBlockType()
// These IOs are used for routing in the parent
io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq())
io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && !io.irel().isVoluntary()
io.matches.oprb := Bool(false)
val outerParams = p.alterPartial({ case TLId => outerTLId })
val oacq_probe = PutBlock(
client_xact_id = UInt(trackerId),
addr_block = io.irel().addr_block,
addr_beat = io.irel().addr_beat,
data = io.irel().data)(outerParams)
val oacq_write_beat = Put(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = xact.addr_beat,
data = xact.data_buffer(0),
wmask = xact.wmask())(outerParams)
val oacq_write_block = PutBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = oacq_data_cnt,
data = xact.data_buffer(oacq_data_cnt),
wmask = xact.wmask_buffer(oacq_data_cnt))(outerParams)
val oacq_read_beat = Get(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block,
addr_beat = xact.addr_beat,
addr_byte = xact.addr_byte(),
operand_size = xact.op_size(),
alloc = Bool(false))(outerParams)
val oacq_read_block = GetBlock(
client_xact_id = UInt(trackerId),
addr_block = xact.addr_block)(outerParams)
io.outer.acquire.valid := Bool(false)
io.outer.acquire.bits := Mux(state === s_probe, oacq_probe,
Mux(state === s_mem_write,
Mux(subblock_type, oacq_write_beat, oacq_write_block),
Mux(subblock_type, oacq_read_beat, oacq_read_block)))
io.outer.grant.ready := Bool(false)
io.inner.probe.valid := Bool(false)
io.inner.probe.bits := coh.makeProbe(curr_p_id, xact)
io.inner.grant.valid := Bool(false)
io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent
io.inner.acquire.ready := Bool(false)
io.inner.release.ready := Bool(false)
io.inner.finish.ready := Bool(false)
assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() &&
io.iacq().client_id =/= xact.client_id),
"AcquireTracker accepted data beat from different network source than initial request.")
assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() &&
io.iacq().client_xact_id =/= xact.client_xact_id),
"AcquireTracker accepted data beat from different client transaction than initial request.")
assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq &&
io.iacq().hasMultibeatData() && io.iacq().addr_beat =/= UInt(0)),
"AcquireTracker initialized with a tail data beat.") "AcquireTracker initialized with a tail data beat.")
when(collect_iacq_data) { assert(!(state =/= s_idle && pending_ignt && xact_iacq.isPrefetch()),
io.inner.acquire.ready := Bool(true) "Broadcast Hub does not support Prefetches.")
when(io.inner.acquire.valid) {
xact.data_buffer(io.iacq().addr_beat) := io.iacq().data assert(!(state =/= s_idle && pending_ignt && xact_iacq.isAtomic()),
xact.wmask_buffer(io.iacq().addr_beat) := io.iacq().wmask() "Broadcast Hub does not support PutAtomics.")
iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true))
}
when(iacq_data_done) { collect_iacq_data := Bool(false) }
} }
when(pending_ognt_ack) { class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
io.outer.grant.ready := Bool(true) extends BroadcastVoluntaryReleaseTracker(trackerId)(p)
when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } with HasDataBuffer {
//TODO add finish queue if this isnt the last level manager
// Tell the parent if any incoming messages conflict with the ongoing transaction
routeInParent()
io.alloc_iacq.can := Bool(false)
// Start transaction by accepting inner release
innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
// A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact
when(irel_is_allocating) { pending_orel := io.irel().hasData() }
when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data }
// Dispatch outer release
outerRelease(
coh = outer_coh.onHit(M_XWR),
data = data_buffer(vol_ognt_counter.up.idx))
quiesce()
} }
switch (state) { class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
is(s_idle) { extends BroadcastAcquireTracker(trackerId)(p)
io.inner.acquire.ready := Bool(true) with HasByteWriteMaskBuffer {
when(io.inner.acquire.valid && io.alloc.iacq) {
xact := io.iacq() // Setup IOs used for routing in the parent
xact.data_buffer(UInt(0)) := io.iacq().data routeInParent()
xact.wmask_buffer(UInt(0)) := io.iacq().wmask() io.alloc_irel.can := Bool(false)
collect_iacq_data := io.iacq().hasMultibeatData()
iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat // First, take care of accpeting new acquires or secondary misses
val needs_probes = mask_incoherent.orR // Handling of primary and secondary misses' data and write mask merging
when(needs_probes) { innerAcquire(
pending_probes := mask_incoherent can_alloc = Bool(false),
release_count := PopCount(mask_incoherent) next = s_inner_probe)
}
state := Mux(needs_probes, s_probe, io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact
Mux(pending_outer_write_, s_mem_write,
Mux(pending_outer_read_, s_mem_read, s_make_grant))) // Track which clients yet need to be probed and make Probe message
} // If a writeback occurs, we can forward its data via the buffer,
} // and skip having to go outwards
is(s_probe) { val skip_outer_acquire = pending_ignt_data.andR
// Generate probes
io.inner.probe.valid := pending_probes.orR innerProbe(
when(io.inner.probe.ready) { inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
pending_probes := pending_probes & ~UIntToOH(curr_p_id) Mux(!skip_outer_acquire, s_outer_acquire, s_busy))
// Handle incoming releases from clients, which may reduce sharer counts
// and/or write back dirty data, and may be unexpected voluntary releases
def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
io.irel().isVoluntary() &&
!Vec(s_idle, s_meta_write).contains(state) &&
!all_pending_done &&
!io.outer.grant.fire() &&
!io.inner.grant.fire() &&
!vol_ignt_counter.pending
innerRelease(block_vol_ignt = vol_ognt_counter.pending)
//TODO: accept vol irels when state === s_idle, operate like the VolRelTracker
io.inner.release.ready := irel_can_merge || irel_same_xact
mergeDataInner(io.inner.release)
// If there was a writeback, forward it outwards
outerRelease(
coh = outer_coh.onHit(M_XWR),
data = data_buffer(vol_ognt_counter.up.idx))
// Send outer request for miss
outerAcquire(
caching = !xact_iacq.isBuiltInType(),
coh = outer_coh,
data = data_buffer(ognt_counter.up.idx),
wmask = wmask_buffer(ognt_counter.up.idx),
next = s_busy)
// Handle the response from outer memory
mergeDataOuter(io.outer.grant)
// Acknowledge or respond with data
innerGrant(
data = data_buffer(ignt_data_idx),
external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending)
when(iacq_is_allocating) {
wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle
initializeProbes()
} }
// Handle releases, which may have data to be written back initDataInner(io.inner.acquire)
val matches = io.matches.irel
io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches // Wait for everything to quiesce
when(io.inner.release.valid && matches) { quiesce()
when(io.irel().hasData()) {
io.outer.acquire.valid := Bool(true)
when(io.outer.acquire.ready) {
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
} .otherwise {
release_count := release_count - UInt(1)
when(release_count === UInt(1)) {
state := Mux(pending_outer_write, s_mem_write,
Mux(pending_outer_read, s_mem_read, s_make_grant))
}
}
}
}
is(s_mem_write) { // Write data to outer memory
io.outer.acquire.valid := !pending_ognt_ack && (!collect_iacq_data || iacq_data_valid(oacq_data_cnt))
when(oacq_data_done) {
pending_ognt_ack := Bool(true)
state := Mux(pending_outer_read, s_mem_read, s_mem_resp)
}
}
is(s_mem_read) { // Read data from outer memory (possibly what was just written)
io.outer.acquire.valid := !pending_ognt_ack
when(io.outer.acquire.fire()) { state := s_mem_resp }
}
is(s_mem_resp) { // Wait to forward grants from outer memory
io.outer.grant.ready := io.inner.grant.ready
io.inner.grant.valid := io.outer.grant.valid
when(ignt_data_done) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade)
io.inner.grant.valid := Bool(true)
when(io.inner.grant.ready) {
state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
}
}
is(s_ack) { // Wait for transaction to complete
io.inner.finish.ready := Bool(true)
when(io.inner.finish.valid) { state := s_idle }
}
}
} }

View File

@ -0,0 +1,141 @@
// See LICENSE for license details.
package uncore
import Chisel._
import cde.{Parameters, Field}
class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
// Create TSHRs for outstanding transactions
val irelTrackerList =
(0 until nReleaseTransactors).map(id =>
Module(new BufferlessBroadcastVoluntaryReleaseTracker(id)))
val iacqTrackerList =
(nReleaseTransactors until nTransactors).map(id =>
Module(new BufferlessBroadcastAcquireTracker(id)))
val trackerList = irelTrackerList ++ iacqTrackerList
// Propagate incoherence flags
trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
// Create an arbiter for the one memory port
val outerList = trackerList.map(_.io.outer)
val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
(p.alterPartial({ case TLId => p(OuterTLId) })))
outer_arb.io.in <> outerList
io.outer <> outer_arb.io.out
// Handle acquire transaction initiation
val irel_vs_iacq_conflict =
io.inner.acquire.valid &&
io.inner.release.valid &&
io.irel().conflicts(io.iacq())
doInputRoutingWithAllocation(
in = io.inner.acquire,
outs = trackerList.map(_.io.inner.acquire),
allocs = trackerList.map(_.io.alloc_iacq),
allocOverride = !irel_vs_iacq_conflict)
io.outer.acquire.bits.data := io.inner.acquire.bits.data
io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat
// Handle releases, which might be voluntary and might have data
doInputRoutingWithAllocation(
in = io.inner.release,
outs = trackerList.map(_.io.inner.release),
allocs = trackerList.map(_.io.alloc_irel))
io.outer.release.bits.data := io.inner.release.bits.data
io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat
// Wire probe requests and grant reply to clients, finish acks from clients
doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
io.inner.grant.bits.data := io.outer.grant.bits.data
io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat
doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
}
class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
extends BroadcastVoluntaryReleaseTracker(trackerId)(p) {
// Tell the parent if any incoming messages conflict with the ongoing transaction
routeInParent()
io.alloc_iacq.can := Bool(false)
// Start transaction by accepting inner release
innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
// A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
// and if the outer relase path is clear
val irel_could_accept = state === s_idle || irel_can_merge || irel_same_xact
io.inner.release.ready := irel_could_accept &&
(!io.irel().hasData() || io.outer.release.ready)
// Dispatch outer release
outerRelease(coh = outer_coh.onHit(M_XWR))
io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data
quiesce()
}
class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
extends BroadcastAcquireTracker(trackerId)(p) {
// Setup IOs used for routing in the parent
routeInParent()
io.alloc_irel.can := Bool(false)
// First, take care of accpeting new acquires or secondary misses
// Handling of primary and secondary misses' data and write mask merging
innerAcquire(
can_alloc = Bool(false),
next = s_inner_probe)
val iacq_could_accept = state === s_outer_acquire || iacq_can_merge || iacq_same_xact
io.inner.acquire.ready := iacq_could_accept &&
(!io.iacq().hasData() || io.outer.acquire.fire())
// Track which clients yet need to be probed and make Probe message
innerProbe(
inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
s_outer_acquire)
// Handle incoming releases from clients, which may reduce sharer counts
// and/or write back dirty data, and may be unexpected voluntary releases
def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
io.irel().isVoluntary() &&
!vol_ignt_counter.pending &&
(state =/= s_idle)
innerRelease(block_vol_ignt = vol_ognt_counter.pending)
val irel_could_accept = irel_can_merge || irel_same_xact
io.inner.release.ready := irel_could_accept &&
(!io.irel().hasData() || io.outer.release.ready)
// If there was a writeback, forward it outwards
outerRelease(
coh = outer_coh.onHit(M_XWR),
buffering = Bool(false))
// Send outer request for miss
outerAcquire(
caching = !xact_iacq.isBuiltInType(),
buffering = Bool(false),
coh = outer_coh,
next = s_busy)
// Handle the response from outer memory
io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data
// Acknowledge or respond with data
innerGrant(external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending)
when(iacq_is_allocating) { initializeProbes() }
// Wait for everything to quiesce
quiesce()
}

File diff suppressed because it is too large Load Diff

View File

@ -227,6 +227,17 @@ object ManagerTileLinkHeaderCreator {
} }
} }
class BeatCounterStatus extends Bundle {
val idx = UInt()
val done = Bool()
}
class TwoWayBeatCounterStatus extends Bundle {
val pending = Bool()
val up = new BeatCounterStatus()
val down = new BeatCounterStatus()
}
/** Utility trait containing wiring functions to keep track of how many data beats have /** Utility trait containing wiring functions to keep track of how many data beats have
* been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels.
* *
@ -283,27 +294,31 @@ trait HasDataBeatCounters {
/** Provides counters on two channels, as well a meta-counter that tracks how many /** Provides counters on two channels, as well a meta-counter that tracks how many
* messages have been sent over the up channel but not yet responded to over the down channel * messages have been sent over the up channel but not yet responded to over the down channel
* *
* @param max max number of outstanding ups with no down * @param status bundle of status of the counters
* @param up outgoing channel * @param up outgoing channel
* @param down incoming channel * @param down incoming channel
* @param max max number of outstanding ups with no down
* @param beat overrides cnts on single-beat messages * @param beat overrides cnts on single-beat messages
* @param track whether up's message should be tracked * @param track whether up's message should be tracked
* @return a tuple containing whether their are outstanding messages, up's count, * @return a tuple containing whether their are outstanding messages, up's count,
* up's done, down's count, down's done * up's done, down's count, down's done
*/ */
def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel](
max: Int, status: TwoWayBeatCounterStatus,
up: DecoupledIO[T], up: DecoupledIO[T],
down: DecoupledIO[S], down: DecoupledIO[S],
max: Int = 1,
beat: UInt = UInt(0), beat: UInt = UInt(0),
trackUp: T => Bool = (t: T) => Bool(true), trackUp: T => Bool = (t: T) => Bool(true),
trackDown: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { trackDown: S => Bool = (s: S) => Bool(true)) {
val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat)
val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat)
val do_inc = up_done && trackUp(up.bits) val cnt = TwoWayCounter(up_done, dn_done, max)
val do_dec = down_done && trackDown(down.bits) status.pending := cnt > UInt(0)
val cnt = TwoWayCounter(do_inc, do_dec, max) status.up.idx := up_idx
(cnt > UInt(0), up_idx, up_done, down_idx, down_done) status.up.done := up_done
status.down.idx := dn_idx
status.down.done := dn_done
} }
} }
@ -391,9 +406,10 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) {
addr_beat = ognt.addr_beat, addr_beat = ognt.addr_beat,
data = ognt.data) data = ognt.data)
assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.")
val rel_grant = Grant( val rel_grant = Grant(
is_builtin_type = Bool(true), is_builtin_type = Bool(true),
g_type = Mux(gnt_voluntary, Grant.voluntaryAckType, ognt.g_type), g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases
client_xact_id = ognt.client_xact_id, client_xact_id = ognt.client_xact_id,
manager_xact_id = ognt.manager_xact_id, manager_xact_id = ognt.manager_xact_id,
addr_beat = ognt.addr_beat, addr_beat = ognt.addr_beat,

View File

@ -47,13 +47,14 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) {
def makeAcquire( def makeAcquire(
op_code: UInt, op_code: UInt,
client_xact_id: UInt, client_xact_id: UInt,
addr_block: UInt): Acquire = addr_block: UInt): Acquire = {
Acquire( Acquire(
is_builtin_type = Bool(false), is_builtin_type = Bool(false),
a_type = co.getAcquireType(op_code, this), a_type = co.getAcquireType(op_code, this),
client_xact_id = client_xact_id, client_xact_id = client_xact_id,
addr_block = addr_block, addr_block = addr_block,
union = Cat(op_code, Bool(true)))(p) union = Cat(op_code, Bool(true)))(p)
}
/** Constructs a Release message based on this metadata on cache control op /** Constructs a Release message based on this metadata on cache control op
* *

View File

@ -0,0 +1,118 @@
// See LICENSE for license details.
package uncore
import Chisel._
import cde.{Parameters, Field}
case object L2StoreDataQueueDepth extends Field[Int]
trait HasStoreDataQueueParameters extends HasCoherenceAgentParameters {
val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats
val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
}
class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p)
with HasStoreDataQueueParameters {
val idx = UInt(width = dqIdxBits)
val loc = UInt(width = log2Ceil(nDataQueueLocations))
}
object DataQueueLocation {
def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = {
val d = Wire(new DataQueueLocation)
d.idx := idx
d.loc := loc
d
}
}
trait HasStoreDataQueue extends HasStoreDataQueueParameters {
val io: HierarchicalTLIO
val trackerIOsList: Seq[HierarchicalXactTrackerIO]
val internalDataBits = new DataQueueLocation().getWidth
val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
val usingStoreDataQueue = p.alterPartial({
case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
})
// Queue to store impending Put data
lazy val sdq = Reg(Vec(sdqDepth, io.iacq().data))
lazy val sdq_val = Reg(init=Bits(0, sdqDepth))
lazy val sdq_alloc_id = PriorityEncoder(~sdq_val)
lazy val sdq_rdy = !sdq_val.andR
lazy val sdq_enq = trackerIOsList.map( t =>
(t.alloc_iacq.should || t.alloc_iacq.matches) &&
t.inner.acquire.fire() &&
t.iacq().hasData()
).reduce(_||_)
lazy val sdqLoc = List.fill(nTransactors) {
DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
}
/*
doInputRoutingWithAllocation(
in = io.inner.acquire,
outs = trackerList.map(_.io.inner.acquire),
allocs = trackerList.map(_.io.alloc._iacq),
dataOverride = Some(sdqLoc),
allocOverride = Some(sdq_rdy && !irel_vs_iacq_conflict))
*/
// Queue to store impending Voluntary Release data
lazy val voluntary = io.irel().isVoluntary()
lazy val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData()
lazy val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
lazy val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1
lazy val vwbqLoc = (0 until nTransactors).map(i =>
(DataQueueLocation(rel_data_cnt,
(if(i < nReleaseTransactors) inVolWBQueue
else inClientReleaseQueue)).toBits))
/*
doInputRoutingWithAllocation(
io.inner.release,
trackerList.map(_.io.inner.release),
trackerList.map(_.io.matches.irel),
trackerList.map(_.io.alloc.irel),
Some(vwbqLoc))
*/
val outer_arb: ClientTileLinkIOArbiter
lazy val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data)
/*
val outer_arb = Module(new ClientTileLinkIOArbiter(trackerList.size)
(usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) })))
outer_arb.io.in <> trackerList
*/
// Get the pending data out of the store data queue
lazy val is_in_sdq = outer_data_ptr.loc === inStoreQueue
lazy val free_sdq = io.outer.acquire.fire() &&
io.outer.acquire.bits.hasData() &&
outer_data_ptr.loc === inStoreQueue
/*
io.outer <> outer_arb.io.out
io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array(
inStoreQueue -> sdq(outer_data_ptr.idx),
inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
*/
// Enqueue SDQ data
def sdqEnqueue() {
when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data }
when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data }
}
// Update SDQ valid bits
def sdqUpdate() {
when (io.outer.acquire.valid || sdq_enq) {
sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) |
PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
}
}
}

View File

@ -132,8 +132,8 @@ trait HasTileLinkData extends HasTileLinkBeatId {
def hasData(dummy: Int = 0): Bool def hasData(dummy: Int = 0): Bool
def hasMultibeatData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool
def first(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(0), Bool(true)) def first(dummy: Int = 0): Bool = !hasMultibeatData() || addr_beat === UInt(0)
def last(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(tlDataBeats-1), Bool(true)) def last(dummy: Int = 0): Bool = !hasMultibeatData() || addr_beat === UInt(tlDataBeats-1)
} }
/** An entire cache block of data */ /** An entire cache block of data */
@ -186,6 +186,10 @@ trait HasAcquireUnion extends HasTileLinkParameters {
} }
/** Full, beat-sized writemask */ /** Full, beat-sized writemask */
def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask())
/** Is this message a built-in read message */
def hasPartialWritemask(dummy: Int = 0): Bool = wmask() =/= Acquire.fullWriteMask
} }
trait HasAcquireType extends HasTileLinkParameters { trait HasAcquireType extends HasTileLinkParameters {
@ -207,6 +211,12 @@ trait HasAcquireType extends HasTileLinkParameters {
def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() &&
(is(Acquire.getPrefetchType) || is(Acquire.putPrefetchType)) (is(Acquire.getPrefetchType) || is(Acquire.putPrefetchType))
/** Is this message a built-in atomic message */
def isAtomic(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.putAtomicType)
/** Is this message a built-in read message */
def isGet(dummy: Int = 0): Bool = isBuiltInType() && (is(Acquire.getType) || is(Acquire.getBlockType))
/** Does this message contain data? Assumes that no custom message types have data. */ /** Does this message contain data? Assumes that no custom message types have data. */
def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type)
@ -214,11 +224,6 @@ trait HasAcquireType extends HasTileLinkParameters {
def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() &&
Acquire.typesWithMultibeatData.contains(a_type) Acquire.typesWithMultibeatData.contains(a_type)
/** Does this message require the manager to probe the client the very client that sent it?
* Needed if multiple caches are attached to the same port.
*/
def requiresSelfProbe(dummy: Int = 0) = Bool(false)
/** Mapping between each built-in Acquire type and a built-in Grant type. */ /** Mapping between each built-in Acquire type and a built-in Grant type. */
def getBuiltInGrantType(dummy: Int = 0): UInt = Acquire.getBuiltInGrantType(this.a_type) def getBuiltInGrantType(dummy: Int = 0): UInt = Acquire.getBuiltInGrantType(this.a_type)
} }

View File

@ -0,0 +1,546 @@
// See LICENSE for license details.
package uncore
import Chisel._
import cde.{Parameters, Field}
class TrackerAllocation extends Bundle {
val matches = Bool(OUTPUT)
val can = Bool(OUTPUT)
val should = Bool(INPUT)
}
trait HasTrackerAllocationIO extends Bundle {
val alloc_iacq = new TrackerAllocation
val alloc_irel = new TrackerAllocation
val alloc_oprb = new TrackerAllocation
}
class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p)
with HasTrackerAllocationIO
class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p)
with HasTrackerAllocationIO
abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p)
with HasXactTrackerStates
with HasPendingBitHelpers {
override val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9)
val state = Reg(init=s_idle)
def quiesce(next: UInt = s_idle) {
all_pending_done := !scoreboard.foldLeft(Bool(false))(_||_)
when(state === s_busy && all_pending_done) { state := next }
}
def pinAllReadyValidLow[T <: Data](b: Bundle) {
b.elements.foreach {
_._2 match {
case d: DecoupledIO[_] =>
if(d.ready.dir == OUTPUT) d.ready := Bool(false)
else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false)
case b: Bundle => pinAllReadyValidLow(b)
case _ =>
}
}
}
}
trait HasXactTrackerStates {
def state: UInt
def s_idle: UInt = UInt(0)
def s_meta_read: UInt
def s_meta_resp: UInt
def s_wb_req: UInt
def s_wb_resp: UInt
def s_inner_probe: UInt
def s_outer_acquire: UInt
def s_busy: UInt
def s_meta_write: UInt
}
trait HasPendingBitHelpers extends HasDataBeatCounters {
val scoreboard = scala.collection.mutable.ListBuffer.empty[Bool]
val all_pending_done = Wire(Bool())
def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt =
Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat)
def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt =
~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat)
def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt =
Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id)
def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt =
~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id)
def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt =
addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits)
def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt =
addPendingBitWhenBeatHasData(in, in.bits.allocate())
def addPendingBitWhenBeatNeedsRead(in: DecoupledIO[AcquireFromSrc], inc: Bool = Bool(true)): UInt = {
val a = in.bits
val needs_read = (a.isGet() || a.isAtomic() || a.hasPartialWritemask()) || inc
addPendingBitWhenBeat(in.fire() && needs_read, a)
}
def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt =
addPendingBitWhenBeat(in.fire() && in.bits.hasPartialWritemask(), in.bits)
def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt =
Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat))
def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt =
dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits)
def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt =
dropPendingBitWhenId(in.fire(), in.bits)
def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt =
addPendingBitWhenId(in.fire(), in.bits)
def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
def addOtherBits(en: Bool, nBits: Int): UInt =
Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits))
def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
addOtherBits(in.fire() &&
in.bits.hasMultibeatData() &&
in.bits.addr_beat === UInt(0),
in.bits.tlDataBeats)
def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
~addPendingBitsOnFirstBeat(in)
}
trait HasDataBuffer extends HasCoherenceAgentParameters {
val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits)))
type TLDataBundle = TLBundle with HasTileLinkData with HasTileLinkBeatId
def initDataInner[T <: Acquire](in: DecoupledIO[T]) {
when(in.fire() && in.bits.hasData()) {
data_buffer(in.bits.addr_beat) := in.bits.data
}
}
// TODO: provide func for accessing when innerDataBeats =/= outerDataBeats or internalDataBeats
def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
data_buffer(beat) := incoming
}
def mergeDataInner[T <: TLDataBundle](in: DecoupledIO[T]) {
when(in.fire() && in.bits.hasData()) {
mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data)
}
}
def mergeDataOuter[T <: TLDataBundle](in: DecoupledIO[T]) {
when(in.fire() && in.bits.hasData()) {
mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data)
}
}
}
trait HasByteWriteMaskBuffer extends HasDataBuffer {
val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerWriteMaskBits)))
override def initDataInner[T <: Acquire](in: DecoupledIO[T]) {
when(in.fire() && in.bits.hasData()) {
val beat = in.bits.addr_beat
val full = FillInterleaved(8, in.bits.wmask())
data_buffer(beat) := (~full & data_buffer(beat)) | (full & in.bits.data)
wmask_buffer(beat) := in.bits.wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed
}
}
override def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
val old_data = incoming // Refilled, written back, or de-cached data
val new_data = data_buffer(beat) // Newly Put data is already in the buffer
val wmask = FillInterleaved(8, wmask_buffer(beat))
data_buffer(beat) := ~wmask & old_data | wmask & new_data
}
}
trait HasBlockAddressBuffer extends HasCoherenceAgentParameters {
val xact_addr_block = Reg(init = UInt(0, width = blockAddrBits))
}
trait HasAcquireMetadataBuffer extends HasBlockAddressBuffer {
val xact_allocate = Reg{ Bool() }
val xact_amo_shift_bytes = Reg{ UInt() }
val xact_op_code = Reg{ UInt() }
val xact_addr_byte = Reg{ UInt() }
val xact_op_size = Reg{ UInt() }
val xact_addr_beat = Wire(UInt())
val xact_iacq = Wire(new SecondaryMissInfo)
}
trait HasVoluntaryReleaseMetadataBuffer extends HasBlockAddressBuffer
with HasPendingBitHelpers
with HasXactTrackerStates {
def io: HierarchicalXactTrackerIO
val xact_vol_ir_r_type = Reg{ UInt() }
val xact_vol_ir_src = Reg{ UInt() }
val xact_vol_ir_client_xact_id = Reg{ UInt() }
def xact_vol_irel = Release(
src = xact_vol_ir_src,
voluntary = Bool(true),
r_type = xact_vol_ir_r_type,
client_xact_id = xact_vol_ir_client_xact_id,
addr_block = xact_addr_block)
(p.alterPartial({ case TLId => p(InnerTLId) }))
}
trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
def inner_coh: ManagerMetadata
val pending_irel_data = Reg(init=Bits(0, width = innerDataBeats))
val vol_ignt_counter = Wire(new TwoWayBeatCounterStatus)
def irel_can_merge: Bool
def irel_same_xact: Bool
def irel_is_allocating: Bool = state === s_idle && io.alloc_irel.should && io.inner.release.valid
def irel_is_merging: Bool = (irel_can_merge || irel_same_xact) && io.inner.release.valid
def innerRelease(block_vol_ignt: Bool = Bool(false), next: UInt = s_busy) {
connectTwoWayBeatCounters(
status = vol_ignt_counter,
up = io.inner.release,
down = io.inner.grant,
trackUp = (r: Release) => {
Mux(state === s_idle, io.alloc_irel.should, io.alloc_irel.matches) && r.isVoluntary() && r.requiresAck()
},
trackDown = (g: Grant) => (state =/= s_idle) && g.isVoluntary())
pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release))
when(irel_is_allocating) {
xact_addr_block := io.irel().addr_block
state := next
}
when(io.inner.release.fire()) {
when(io.alloc_irel.should || (irel_can_merge && io.irel().first())) {
xact_vol_ir_r_type := io.irel().r_type
xact_vol_ir_src := io.irel().client_id
xact_vol_ir_client_xact_id := io.irel().client_xact_id
pending_irel_data := Mux(io.irel().hasMultibeatData(),
dropPendingBitWhenBeatHasData(io.inner.release),
UInt(0))
}
}
io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) &&
vol_ignt_counter.pending &&
!(pending_irel_data.orR || block_vol_ignt)
io.inner.grant.bits := inner_coh.makeGrant(xact_vol_irel)
scoreboard += (pending_irel_data.orR, vol_ignt_counter.pending)
}
}
trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
val pending_orel = Reg(init=Bool(false))
val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats))
val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus)
def outerRelease(
coh: ClientMetadata,
buffering: Bool = Bool(true),
data: UInt = io.irel().data,
add_pending_bit: UInt = UInt(0)) {
pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) |
addPendingBitWhenBeatHasData(io.inner.release) |
add_pending_bit
connectTwoWayBeatCounters(
status = vol_ognt_counter,
up = io.outer.release,
down = io.outer.grant,
trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(),
trackDown = (g: Grant) => g.isVoluntary())
io.outer.release.valid := state === s_busy &&
Mux(io.orel().hasData(),
Mux(buffering,
pending_orel_data(vol_ognt_counter.up.idx),
io.inner.release.valid),
pending_orel)
io.outer.release.bits := coh.makeVoluntaryWriteback(
client_xact_id = UInt(0), // TODO was tracker id, but not needed?
addr_block = xact_addr_block,
addr_beat = vol_ognt_counter.up.idx,
data = data)
when(pending_orel_data.orR) { pending_orel := Bool(true) }
when(vol_ognt_counter.up.done) { pending_orel := Bool(false) }
io.outer.grant.ready := state === s_busy
scoreboard += (pending_orel, vol_ognt_counter.pending)
}
}
trait EmitsInnerProbes extends HasBlockAddressBuffer
with HasXactTrackerStates
with HasPendingBitHelpers {
def io: HierarchicalXactTrackerIO
val pending_iprbs = Reg(UInt(width = innerNCachingClients))
val curr_probe_dst = PriorityEncoder(pending_iprbs)
val irel_counter = Wire(new TwoWayBeatCounterStatus)
def full_representation: UInt
def initializeProbes() { pending_iprbs := full_representation & ~io.incoherent.toBits }
def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
!io.irel().isVoluntary() &&
state === s_inner_probe
def innerProbe(prb: Probe, next: UInt) {
pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe)
io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR
io.inner.probe.bits := prb
connectTwoWayBeatCounters(
status = irel_counter,
up = io.inner.probe,
down = io.inner.release,
max = innerNCachingClients,
trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary())
when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) {
state := next
}
//N.B. no pending bits added to scoreboard because all handled in s_inner_probe
}
}
trait RoutesInParent extends HasBlockAddressBuffer
with HasXactTrackerStates {
def io: HierarchicalXactTrackerIO
type AddrComparison = HasCacheBlockAddress => Bool
def exactAddrMatch(a: HasCacheBlockAddress): Bool = a.conflicts(xact_addr_block)
def routeInParent(iacqMatches: AddrComparison = exactAddrMatch,
irelMatches: AddrComparison = exactAddrMatch,
oprbMatches: AddrComparison = exactAddrMatch) {
io.alloc_iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq())
io.alloc_irel.matches := (state =/= s_idle) && irelMatches(io.irel())
io.alloc_oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb())
io.alloc_iacq.can := state === s_idle
io.alloc_irel.can := state === s_idle
io.alloc_oprb.can := Bool(false)
}
}
trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer
with AcceptsVoluntaryReleases
with HasXactTrackerStates
with HasPendingBitHelpers {
def io: HierarchicalXactTrackerIO
def nSecondaryMisses: Int
def alwaysWriteFullBeat: Boolean
def inner_coh: ManagerMetadata
def trackerId: Int
// Secondary miss queue holds transaction metadata used to make grants
lazy val ignt_q = Module(new Queue(
new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })),
1 + nSecondaryMisses))
val pending_ignt = Wire(Bool())
val ignt_data_idx = Wire(UInt())
val ignt_data_done = Wire(Bool())
val ifin_counter = Wire(new TwoWayBeatCounterStatus)
val pending_put_data = Reg(init=Bits(0, width = innerDataBeats))
val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats))
def iacq_same_xact: Bool = {
(xact_iacq.client_xact_id === io.iacq().client_xact_id) &&
xact_iacq.hasMultibeatData() &&
pending_ignt &&
pending_put_data(io.iacq().addr_beat)
}
def iacq_can_merge: Bool
def iacq_is_allocating: Bool = state === s_idle && io.alloc_iacq.should && io.inner.acquire.valid
def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid
def innerAcquire(can_alloc: Bool, next: UInt) {
// Enqueue some metadata information that we'll use to make coherence updates with later
ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && !iacq_same_xact)
ignt_q.io.enq.bits := io.iacq()
// Use the outputs of the queue to make further messages
xact_iacq := Mux(ignt_q.io.deq.valid, ignt_q.io.deq.bits, ignt_q.io.enq.bits)
xact_addr_beat := xact_iacq.addr_beat
pending_ignt := ignt_q.io.count > UInt(0)
// Track whether any beats are missing from a PutBlock
pending_put_data := (pending_put_data &
dropPendingBitWhenBeatHasData(io.inner.acquire)) |
addPendingBitsOnFirstBeat(io.inner.acquire)
// Intialize transaction metadata for accepted Acquire
when(iacq_is_allocating) {
xact_addr_block := io.iacq().addr_block
xact_allocate := io.iacq().allocate() && can_alloc
xact_amo_shift_bytes := io.iacq().amo_shift_bytes()
xact_op_code := io.iacq().op_code()
xact_addr_byte := io.iacq().addr_byte()
xact_op_size := io.iacq().op_size()
// Make sure to collect all data from a PutBlock
pending_put_data := Mux(
io.iacq().isBuiltInType(Acquire.putBlockType),
dropPendingBitWhenBeatHasData(io.inner.acquire),
UInt(0))
pending_ignt_data := UInt(0)
state := next
}
scoreboard += (pending_put_data.orR)
}
def innerGrant(
data: UInt = io.ognt().data,
external_pending: Bool = Bool(false),
add: UInt = UInt(0)) {
// Track the number of outstanding inner.finishes
connectTwoWayBeatCounters(
status = ifin_counter,
up = io.inner.grant,
down = io.inner.finish,
max = nSecondaryMisses,
trackUp = (g: Grant) => g.requiresAck())
// Track which beats are ready for response
when(!iacq_is_allocating) {
pending_ignt_data := (pending_ignt_data & dropPendingBitWhenBeatHasData(io.inner.grant)) |
addPendingBitWhenBeatHasData(io.inner.release) |
addPendingBitWhenBeatHasData(io.outer.grant) |
add
}
// We can issue a grant for a pending write once all data is
// received and committed to the data array or outer memory
val ignt_ack_ready = !(state === s_idle ||
state === s_meta_read ||
pending_put_data.orR)
val ignt_from_iacq = inner_coh.makeGrant(
sec = ignt_q.io.deq.bits,
manager_xact_id = UInt(trackerId),
data = data)
// Make the Grant message using the data stored in the secondary miss queue
val (cnt, done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat)
ignt_data_idx := cnt
ignt_data_done := done
ignt_q.io.deq.ready := Bool(false)
when(!vol_ignt_counter.pending) {
ignt_q.io.deq.ready := ignt_data_done
io.inner.grant.bits := ignt_from_iacq
io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter
when (state === s_busy && pending_ignt) {
io.inner.grant.valid := !external_pending &&
Mux(io.ignt().hasData(), pending_ignt_data(ignt_data_idx), ignt_ack_ready)
}
}
// We must wait for as many Finishes as we sent Grants
io.inner.finish.ready := state === s_busy
scoreboard += (pending_ignt, ifin_counter.pending)
}
}
trait EmitsOuterAcquires extends AcceptsInnerAcquires {
val ognt_counter = Wire(new TwoWayBeatCounterStatus)
// Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory:
//
// If we're allocating in this cache, we can use the current metadata
// to make an appropriate custom Acquire, otherwise we copy over the
// built-in Acquire from the inner TL to the outer TL
def outerAcquire(
caching: Bool,
coh: ClientMetadata,
buffering: Bool = Bool(true),
data: UInt = io.iacq().data,
wmask: UInt = io.iacq().wmask(),
next: UInt = s_busy) {
// Tracks outstanding Acquires, waiting for their matching Grant.
connectTwoWayBeatCounters(
status = ognt_counter,
up = io.outer.acquire,
down = io.outer.grant,
beat = xact_addr_beat,
trackDown = (g: Grant) => !g.isVoluntary())
io.outer.acquire.valid := state === s_outer_acquire &&
(xact_allocate ||
Mux(buffering,
!pending_put_data(ognt_counter.up.idx),
io.inner.acquire.valid))
io.outer.acquire.bits :=
Mux(caching,
coh.makeAcquire(
op_code = xact_op_code,
client_xact_id = UInt(0),
addr_block = xact_addr_block),
BuiltInAcquireBuilder(
a_type = xact_iacq.a_type,
client_xact_id = UInt(0),
addr_block = xact_addr_block,
addr_beat = ognt_counter.up.idx,
data = data,
addr_byte = xact_addr_byte,
operand_size = xact_op_size,
opcode = xact_op_code,
wmask = wmask,
alloc = Bool(false))
(p.alterPartial({ case TLId => p(OuterTLId)})))
when(state === s_outer_acquire && ognt_counter.up.done) { state := next }
io.outer.grant.ready := state === s_busy
scoreboard += ognt_counter.pending
}
}
abstract class VoluntaryReleaseTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
with AcceptsVoluntaryReleases
with RoutesInParent {
def irel_can_merge = Bool(false)
def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
io.irel().isVoluntary() &&
pending_irel_data.orR
}
abstract class AcquireTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
with AcceptsInnerAcquires
with EmitsOuterAcquires
with EmitsInnerProbes
with RoutesInParent {
}

View File

@ -3,6 +3,7 @@
package uncore package uncore
import Chisel._ import Chisel._
import cde.{Parameters, Field} import cde.{Parameters, Field}
import junctions._
case object NReleaseTransactors extends Field[Int] case object NReleaseTransactors extends Field[Int]
case object NProbeTransactors extends Field[Int] case object NProbeTransactors extends Field[Int]
@ -18,6 +19,7 @@ trait HasCoherenceAgentParameters {
val nReleaseTransactors = 1 val nReleaseTransactors = 1
val nAcquireTransactors = p(NAcquireTransactors) val nAcquireTransactors = p(NAcquireTransactors)
val nTransactors = nReleaseTransactors + nAcquireTransactors val nTransactors = nReleaseTransactors + nAcquireTransactors
val blockAddrBits = p(PAddrBits) - p(CacheBlockOffsetBits)
val outerTLId = p(OuterTLId) val outerTLId = p(OuterTLId)
val outerTLParams = p(TLKey(outerTLId)) val outerTLParams = p(TLKey(outerTLId))
val outerDataBeats = outerTLParams.dataBeats val outerDataBeats = outerTLParams.dataBeats
@ -32,6 +34,7 @@ trait HasCoherenceAgentParameters {
val innerWriteMaskBits = innerTLParams.writeMaskBits val innerWriteMaskBits = innerTLParams.writeMaskBits
val innerBeatAddrBits = log2Up(innerDataBeats) val innerBeatAddrBits = log2Up(innerDataBeats)
val innerByteAddrBits = log2Up(innerDataBits/8) val innerByteAddrBits = log2Up(innerDataBits/8)
val innerNCachingClients = innerTLParams.nCachingClients
val maxManagerXacts = innerTLParams.maxManagerXacts val maxManagerXacts = innerTLParams.maxManagerXacts
require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement
} }
@ -66,31 +69,33 @@ trait HasCoherenceAgentWiringHelpers {
* *
* When a match is reported, if ready is high the new transaction * When a match is reported, if ready is high the new transaction
* is merged; when ready is low the transaction is being blocked. * is merged; when ready is low the transaction is being blocked.
* When no match is reported, any high readys are presumed to be * When no match is reported, any high idles are presumed to be
* from trackers that are available for allocation, and one is * from trackers that are available for allocation, and one is
* assigned via alloc based on priority; f no readys are high then * assigned via alloc based on priority; if no idles are high then
* all trackers are busy with other transactions. * all trackers are busy with other transactions. If idle is high
* but ready is low, the tracker will be allocated but does not
* have sufficient buffering for the data.
*/ */
def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData]( def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData](
in: DecoupledIO[T], in: DecoupledIO[T],
outs: Seq[DecoupledIO[T]], outs: Seq[DecoupledIO[T]],
matches: Seq[Bool], allocs: Seq[TrackerAllocation],
allocs: Seq[Bool],
dataOverrides: Option[Seq[UInt]] = None, dataOverrides: Option[Seq[UInt]] = None,
allocOverride: Option[Bool] = None, allocOverride: Option[Bool] = None,
matchOverride: Option[Bool] = None) { matchOverride: Option[Bool] = None) {
val ready_bits = Vec(outs.map(_.ready)).toBits val ready_bits = Vec(outs.map(_.ready)).toBits
val alloc_bits = PriorityEncoderOH(ready_bits) val can_alloc_bits = Vec(allocs.map(_.can)).toBits
val match_bits = Vec(matches).toBits val should_alloc_bits = PriorityEncoderOH(can_alloc_bits)
val match_bits = Vec(allocs.map(_.matches)).toBits
val no_matches = !match_bits.orR val no_matches = !match_bits.orR
val alloc_ok = allocOverride.getOrElse(Bool(true)) val alloc_ok = allocOverride.getOrElse(Bool(true))
val match_ok = matchOverride.getOrElse(Bool(true)) val match_ok = matchOverride.getOrElse(Bool(true))
in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && alloc_ok && match_ok in.ready := (Mux(no_matches, can_alloc_bits, match_bits) & ready_bits).orR && alloc_ok && match_ok
outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => outs.zip(allocs).zipWithIndex.foreach { case((out, alloc), i) =>
out.valid := in.valid && match_ok && alloc_ok out.valid := in.valid && match_ok && alloc_ok
out.bits := in.bits out.bits := in.bits
dataOverrides foreach { d => out.bits.data := d(i) } dataOverrides foreach { d => out.bits.data := d(i) }
a := alloc_bits(i) & no_matches & alloc_ok alloc.should := should_alloc_bits(i) && no_matches && alloc_ok
} }
} }
} }
@ -141,87 +146,10 @@ class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p)
with HasInnerTLIO with HasInnerTLIO
with HasCachedOuterTLIO with HasCachedOuterTLIO
abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) { abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p)
with HasCoherenceAgentWiringHelpers {
val io = new HierarchicalTLIO val io = new HierarchicalTLIO
def innerTL = io.inner def innerTL = io.inner
def outerTL = io.outer def outerTL = io.outer
def incoherent = io.incoherent def incoherent = io.incoherent
} }
trait HasTrackerAllocationIO extends Bundle {
val matches = new Bundle {
val iacq = Bool(OUTPUT)
val irel = Bool(OUTPUT)
val oprb = Bool(OUTPUT)
}
val alloc = new Bundle {
val iacq = Bool(INPUT)
val irel = Bool(INPUT)
val oprb = Bool(INPUT)
}
}
class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p)
with HasTrackerAllocationIO
class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p)
with HasTrackerAllocationIO
abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p)
with HasDataBeatCounters {
def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt =
Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat)
def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt =
~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat)
def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt =
Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id)
def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt =
~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id)
def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt =
addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits)
def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt =
addPendingBitWhenBeatHasData(in, in.bits.allocate())
def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = {
val a = in.bits
val isGetOrAtomic = a.isBuiltInType() &&
(Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type))
addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a)
}
def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt =
Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat))
def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt =
dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits)
def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt =
dropPendingBitWhenId(in.fire(), in.bits)
def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt =
addPendingBitWhenId(in.fire(), in.bits)
def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
def pinAllReadyValidLow[T <: Data](b: Bundle) {
b.elements.foreach {
_._2 match {
case d: DecoupledIO[_] =>
if(d.ready.dir == OUTPUT) d.ready := Bool(false)
else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false)
case b: Bundle => pinAllReadyValidLow(b)
case _ =>
}
}
}
}