From 1bed6ea498de86d51815b0c8b95f861bbe8f6ad0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 28 Feb 2015 17:02:13 -0800 Subject: [PATCH] New metadata-based coherence API --- uncore/src/main/scala/broadcast.scala | 380 ++++++++ uncore/src/main/scala/cache.scala | 1236 ++++++++++++------------- uncore/src/main/scala/coherence.scala | 985 +++++++++----------- uncore/src/main/scala/directory.scala | 43 + uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/memserdes.scala | 314 +++++-- uncore/src/main/scala/metadata.scala | 188 ++++ uncore/src/main/scala/tilelink.scala | 307 +++--- uncore/src/main/scala/uncore.scala | 508 +++------- 9 files changed, 2117 insertions(+), 1851 deletions(-) create mode 100644 uncore/src/main/scala/broadcast.scala create mode 100644 uncore/src/main/scala/directory.scala create mode 100644 uncore/src/main/scala/metadata.scala diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala new file mode 100644 index 00000000..bd8a3f24 --- /dev/null +++ b/uncore/src/main/scala/broadcast.scala @@ -0,0 +1,380 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object L2StoreDataQueueDepth extends Field[Int] + +trait BroadcastHubParameters extends CoherenceAgentParameters { + val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation extends Bundle with BroadcastHubParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + +class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent + with BroadcastHubParameters { + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + // Create SHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + + // Propagate incoherence flags + trackerList.map(_.io.incoherent := io.incoherent.toBits) + + // Queue to store impending Put data + val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + + // Queue to store impending Voluntary Release data + val release = io.inner.release + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits + t.release.valid := release.valid && (release_idx === UInt(i)) + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Wire probe requests and grant reply to clients, finish acks from clients + // Note that we bypass the Grant data subbundles + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + { case TLId => params(OuterTLId) + case TLDataBits => internalDataBits }) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + // Get the pending data out of the store data queue + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.payload.hasData() && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } +} + +class BroadcastXactTracker extends XactTracker { + val io = new ManagerXactTrackerIO +} + +class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = ManagerMetadata.onReset + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + + io.has_acquire_conflict := Bool(false) + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + + io.outer.acquire.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + + //TODO: Use io.outer.release instead? + io.outer.acquire.bits.payload := Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(UInt(0)) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := Mux(io.irel().hasData(), s_outer, + Mux(io.irel().requiresAck(), s_ack, s_idle)) + } + } + is(s_outer) { + io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) + when(oacq_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { // Forward the Grant.voluntaryAck + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(io.inner.grant.fire()) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val coh = ManagerMetadata.onReset + + assert(!(state != s_idle && xact.isBuiltInType() && + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), + UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.incoherent.toBits | myflag) + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ognt_ack = Reg(init=Bool(false)) + val pending_outer_write = xact.hasData() + val pending_outer_read = io.ignt().hasData() + val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + + io.has_acquire_conflict := xact.conflicts(io.iacq()) && + !collect_iacq_data && + (state != s_idle) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state != s_idle) + io.has_acquire_match := Bool(false) + + val outer_write_acq = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + val outer_write_rel = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = io.irel().addr_beat, + data = io.irel().data))(outerTLParams) + val outer_read = Bundle(GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block))(outerTLParams) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default + io.outer.grant.ready := Bool(false) + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := coh.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + when(pending_ognt_ack) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } + //TODO add finish queue if this isnt the last level manager + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(UInt(0)) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + probe_flags := probe_initial_flags + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(pending_outer_write_, s_mem_write, + Mux(pending_outer_read_, s_mem_read, s_make_grant))) + } + } + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready + when(io.inner.release.valid) { + when(io.irel().hasData()) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } .otherwise { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_write) { // Write data to outer memory + io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.bits.payload := outer_write_acq + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + state := Mux(pending_outer_read, s_mem_read, s_mem_resp) + } + } + is(s_mem_read) { // Read data from outer memory (possibly what was just written) + io.outer.acquire.valid := !pending_ognt_ack + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.fire()) { state := s_mem_resp } + } + is(s_mem_resp) { // Wait to forward grants from outer memory + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(ignt_data_done) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4380e255..75eb897d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,7 +2,6 @@ package uncore import Chisel._ -import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -11,6 +10,7 @@ case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -140,7 +140,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(nWays){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) @@ -148,17 +147,14 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } + val metabits = rstVal.getWidth val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until nWays) { - val m = tags(metabits*(w+1)-1, metabits*w) - io.resp(w) := rstVal.clone.fromBits(m) - } - + io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } @@ -166,35 +162,17 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = tlDataBits/rowBits - val refillCycles = refillCyclesPerBeat*tlDataBeats + val blockAddrBits = params(TLBlockAddrBits) + val refillCyclesPerBeat = outerDataBits/rowBits + val refillCycles = refillCyclesPerBeat*outerDataBeats require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) - require(amoAluOperandBits <= tlDataBits) + require(amoAluOperandBits <= innerDataBits) + require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states } -abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters - -abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { - def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { - val (cnt, cnt_done) = - Counter(inc && data.hasMultibeatData(), tlDataBeats) - val done = (inc && !data.hasMultibeatData()) || cnt_done - (cnt, done) - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload)._2 - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { - connectDataBeatCounter(in.fire(), in.bits) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { - connectDataBeatCounter(in.valid, in.bits)._2 - } -} +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -206,22 +184,29 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } -trait HasL2Data extends HasTileLinkData { +trait HasL2BeatAddr extends L2HellaCacheBundle { + val addr_beat = UInt(width = log2Up(refillCycles)) +} + +trait HasL2Data extends L2HellaCacheBundle + with HasL2BeatAddr { + val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) + def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) +} + +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: ManagerMetadata) = { + def apply(tag: Bits, coh: HierarchicalMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new ManagerMetadata -} class L2MetaReadReq extends MetaReadReq with HasL2Id { val tag = Bits(width = tagBits) @@ -250,7 +235,8 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) + def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) + val meta = Module(new MetadataArray(onReset _)) meta.io.read <> io.read meta.io.write <> io.write @@ -259,7 +245,7 @@ class L2MetadataArray extends L2HellaCacheModule { def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) @@ -281,7 +267,7 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasTileLinkBeatId + with HasL2BeatAddr with HasL2Id { val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) @@ -289,7 +275,7 @@ class L2DataReadReq extends L2HellaCacheBundle class L2DataWriteReq extends L2DataReadReq with HasL2Data { - val wmask = Bits(width = tlWriteMaskBits) + val wmask = Bits(width = rowBits/8) } class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data @@ -305,7 +291,7 @@ trait HasL2DataWriteIO extends L2HellaCacheBundle { class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO -class L2DataArray extends L2HellaCacheModule { +class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip val wmask = FillInterleaved(8, io.write.bits.wmask) @@ -320,23 +306,22 @@ class L2DataArray extends L2HellaCacheModule { reg_raddr := raddr } - io.resp.valid := ShiftRegister(io.read.fire(), 1) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) - io.resp.bits.data := array(reg_raddr) + io.resp.valid := ShiftRegister(io.read.fire(), delay+1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) + io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) io.read.ready := !io.write.valid io.write.ready := Bool(true) } -class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - +class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent + with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) + val data = Module(new L2DataArray(1)) tshrfile.io.inner <> io.inner tshrfile.io.meta <> meta.io @@ -345,45 +330,28 @@ class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends io.incoherent <> tshrfile.io.incoherent } +class TSHRFileIO extends HierarchicalTLIO { + val meta = new L2MetaRWIO + val data = new L2DataRWIO +} -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta = new L2MetaRWIO - val data = new L2DataRWIO - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { - val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } - } +class TSHRFile(bankId: Int) extends L2HellaCacheModule + with HasCoherenceAgentWiringHelpers { + val io = new TSHRFileIO // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + Module(new L2VoluntaryReleaseTracker(id, bankId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + Module(new L2AcquireTracker(id, bankId)) } - val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire @@ -414,29 +382,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(releaseList.map(_.ready)).read(release_idx) - // Wire finished transaction acks - val finish = io.inner.finish - val finish_idx = finish.bits.payload.manager_xact_id - trackerList.zipWithIndex.map { case (t, i) => - t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) - } - trackerList.map(_.io.inner.finish.bits := finish.bits) - finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - - // Wire probe requests to clients + // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) - - // Wire grant reply to initiating client - doOutputArbitration( - io.inner.grant, - trackerList.map(_.io.inner.grant), - tlDataBeats, - (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), - {case TLId => outerId}) + val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -449,10 +402,495 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } + +class L2XactTrackerIO extends HierarchicalXactTrackerIO { + val data = new L2DataRWIO + val meta = new L2MetaRWIO + val wb = new L2WritebackIO +} + +abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { + if(data.refillCycles > 1) { + val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) + } else { (UInt(0), inc) } + } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + in: DecoupledIO[T], + beat: UInt = UInt(0), + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.fire(), in.bits, beat, full_block) + } + def connectInternalDataBeatCounter[T <: HasL2Data]( + in: ValidIO[T], + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 + } +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = xact_meta.coh + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := io.irel().isVoluntary() + + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.wb.req.valid := Bool(false) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(io.irel().addr_beat) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := Mux(io.meta.resp.bits.tag_match, + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_inner_grant, s_idle)) + } + } + is(s_data_write) { + io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? + } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + Reg(io.iacq().data.clone) + } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val pending_coh = Reg{ xact_meta.coh.clone } + val pending_finish = Reg{ io.outer.finish.bits.clone } + + val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val do_allocate = xact.allocate() + val needs_writeback = !xact_tag_match && do_allocate && + xact_meta.coh.outer.requiresVoluntaryWriteback() + val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src), + pending_coh.outer.onHit(M_XWR)) // WB is a write + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.ognt(), xact.op_code())) + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val mask_self = Mux(xact.requiresSelfProbe(), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := data_buffer(0) // default + + // TODO: figure out how to merge the following three versions of this func + def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) + def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + + //TODO: Allow hit under miss for stores + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + io.iacq().addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := !io.irel().isVoluntary() && + (xact.addr_block === io.irel().addr_block) && + (state === s_probe) + + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Mux(do_allocate, + xact_meta.coh.outer.makeAcquire( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_cnt, + data = Mux(xact.is(Acquire.putAtomicType), + data_buffer(innerDataBeats), + data_buffer(ignt_data_cnt))) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.read.bits.addr_beat := read_data_cnt + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh := pending_coh + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(io.iacq().addr_beat) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + irel_had_data := Bool(false) + ognt_had_data := Bool(false) + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + pending_coh := io.meta.resp.bits.meta.coh + val _coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match + val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _needs_writeback = !_tag_match && do_allocate && + _coh.outer.requiresVoluntaryWriteback() + val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + when(_is_hit) { pending_coh := pending_coh_on_hit } + when(_needs_probes) { + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + } + state := Mux(_tag_match, + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + } + } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_acquire } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := pending_probes != UInt(0) + when(io.inner.probe.ready) { + pending_probes := pending_probes & ~UIntToOH(curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + pending_coh := pending_coh_on_irel + // Handle released dirty data + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! + when(io.irel().hasData()) { + irel_had_data := Bool(true) + mergeDataInner(data_buffer, io.irel()) + } + // We don't decrement release_count until we've received all the data beats. + when(!io.irel().hasMultibeatData() || irel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + } + } + is(s_outer_acquire) { + io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + when(oacq_data_done) { + state := s_outer_grant + } + } + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().hasData()) { + mergeDataOuter(data_buffer, io.ognt()) + ognt_had_data := Bool(true) + } + when(ognt_data_done) { + pending_coh := pending_coh_on_ognt + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := Mux(!do_allocate, s_inner_grant, + Mux(io.ognt().hasData(), s_data_write, s_data_read)) + } + } + } + } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { + state := Mux(!do_allocate, s_inner_grant, + Mux(ognt_had_data, s_data_write, s_data_read)) + } + } + is(s_data_read) { + io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(read_data_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(resp_data_done) { + state := Mux(xact.hasData(), s_data_write, s_inner_grant) + } + } + is(s_data_write) { + io.data.write.valid := Bool(true) + when(write_data_done) { state := s_inner_grant } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(ignt_data_done) { + val meta = pending_coh_on_ignt != xact_meta.coh + when(meta) { pending_coh := pending_coh_on_ignt } + state := Mux(meta, s_meta_write, + Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr_block = UInt(width = tlBlockAddrBits) - val coh = new ManagerMetadata + val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size + val coh = new HierarchicalMetadata val way_en = Bits(width = nWays) } @@ -463,58 +901,63 @@ class L2WritebackIO extends L2HellaCacheBundle { val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val wb = new L2WritebackIO().flip - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - } - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val mgnt = io.outer.grant.bits +class L2WritebackUnitIO extends HierarchicalXactTrackerIO { + val wb = new L2WritebackIO().flip + val data = new L2DataRWIO +} - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) +class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2WritebackUnitIO + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_coh = Reg{ new ManagerMetadata } + val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) + val xact_coh = Reg{ new HierarchicalMetadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } + val pending_finish = Reg{ io.outer.finish.bits.clone } - val crel_had_data = Reg(init = Bool(false)) + val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) - val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - io.has_release_match := !crel.payload.isVoluntary() && - crel.payload.conflicts(xact_addr_block) && + val pending_icoh_on_irel = xact_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := !io.irel().isVoluntary() && + io.irel().conflicts(xact_addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) // default + io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( client_xact_id = UInt(trackerId), addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - io.outer.grant.ready := Bool(false) // Never gets mgnts + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) + io.outer.release.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // default + io.outer.finish.valid := Bool(false) // default + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) + io.inner.probe.bits.payload := + xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -540,528 +983,75 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id + irel_had_data := Bool(false) val coh = io.wb.req.bits.coh - val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - pending_probes := mask_incoherent - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) + val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) } state := Mux(needs_probes, s_probe, s_data_read) } } is(s_probe) { // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) + io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_coh := next_coh_on_rel + xact_coh.inner := pending_icoh_on_irel // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - xact_data(crel.payload.addr_beat) := crel.payload.data + when(io.irel().hasData()) { + irel_had_data := Bool(true) + data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasData() || crel_data_done) { + when(!io.irel().hasData() || irel_data_done) { release_count := release_count - UInt(1) } } when(release_count === UInt(0)) { - state := Mux(crel_had_data, s_outer_write, s_data_read) + state := Mux(irel_had_data, s_outer_release, s_data_read) } } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_write } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_release } } - is(s_outer_write) { - io.outer.acquire.valid := Bool(true) - when(macq_data_done) { - io.wb.resp.valid := Bool(true) - state := s_idle + is(s_outer_release) { + io.outer.release.valid := Bool(true) + when(orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) } } - } -} - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - val meta = new L2MetaRWIO - val wb = new L2WritebackIO - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cack = io.inner.finish.bits - val mgnt = io.outer.grant.bits -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_crel_data = Reg(init=Bool(false)) - val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) - - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, - xact_meta.coh, - xact_src) - io.wb.req.valid := Bool(false) - - when(collect_crel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(crel.payload.addr_beat) := crel.payload.data - crel_data_valid(crel.payload.addr_beat) := Bool(true) - } - when(crel_data_done) { collect_crel_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_addr_beat := crel.payload.addr_beat - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_crel_data := crel.payload.hasMultibeatData() - crel_data_valid := Bits(1) - state := s_meta_read + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := s_wb_resp + } } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } -} - - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result - Reg(io.inner.acquire.bits.payload.data.clone) - } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) - - val collect_cacq_data = Reg(init=Bool(false)) - val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) - val curr_p_id = co.dir.next(pending_probes) - val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) - val mask_incoherent = mask_self & ~io.tile_incoherent - - val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - - val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit - val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) - val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_builtin_type || xact.allocate() - - val amoalu = Module(new AMOALU) - amoalu.io.addr := xact.addr() - amoalu.io.cmd := xact.op_code() - amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := xact.data(0) // default - - def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.uncachedAtomic), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = tlDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result - } - - //TODO: Allow hit under miss for stores - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - cacq.payload.addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && - (state != s_idle) && - !collect_cacq_data - io.has_acquire_match := xact.conflicts(cacq.payload) && - collect_cacq_data - io.has_release_match := !crel.payload.isVoluntary() && - (xact.addr_block === crel.payload.addr_block) && - (state === s_probe) - - val next_coh_on_rel = co.managerMetadataOnRelease( - incoming = crel.payload, - meta = xact_meta.coh, - src = crel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant( - outgoing = cgnt.payload, - meta = xact_meta.coh, - dst = cgnt.header.dst) - - val outer_write = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := Bool(true) //grant.data -> xact.data - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant( - manager_xact_id = UInt(trackerId), - meta = xact_meta.coh, - addr_beat = cgnt_data_cnt, - data = Mux(xact.is(Acquire.uncachedAtomic), - xact_data(tlDataBeats), - xact_data(cgnt_data_cnt))) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_gnt - - io.wb.req.valid := Bool(false) - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh - io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) - - when(collect_cacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(cacq.payload.addr_beat) := cacq.payload.data - cacq_data_valid(cacq.payload.addr_beat) := Bool(true) - } - when(cacq_data_done) { collect_cacq_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - when( io.inner.acquire.valid ) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_cacq_data := cacq.payload.hasMultibeatData() - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - val coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit - val _needs_probes = _tag_match && co.requiresProbes(xact, coh) - val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.builtin_type || - xact.hasMultibeatData() || - cgnt.payload.hasMultibeatData() - read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) - write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) - resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) - when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) - } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe - } - } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_read } - } - is(s_probe) { - // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) - when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) - } - // Handle releases, which may have data being written back - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_rel - // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - mergeData(xact_data, crel.payload) - } - // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasMultibeatData() || crel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) - } - } - is(s_outer_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := s_outer_resp - } - } - is(s_outer_resp) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - //TODO make sure cacq data is actually present before merging - mergeData(xact_data, mgnt.payload) - when(mgnt_data_done) { - state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) - } - } - } - is(s_data_read) { - io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(io.data.read.ready) { - read_data_cnt := read_data_cnt + UInt(1) - when(read_data_cnt === read_data_max) { state := s_data_resp } - } - } - is(s_data_resp) { - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(resp_data_cnt === resp_data_max) { - state := Mux(xact.hasData(), s_data_write, s_meta_write) - } - } - is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { - write_data_cnt := write_data_cnt + UInt(1) - when(write_data_cnt === write_data_max) { - state := s_meta_write - } - } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - cgnt_data_cnt := cgnt_data_cnt + UInt(1) - when(cgnt_data_cnt === cgnt_data_max) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - } - is(s_ack) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + io.wb.resp.valid := Bool(true) + state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 82f72dea..2524f3d1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,202 +3,148 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers -abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +/** The entire CoherencePolicy API consists of the following three traits: + * HasCustomTileLinkMessageTypes, used to define custom messages + * HasClientSideCoherencePolicy, for client coherence agents + * HasManagerSideCoherencePolicy, for manager coherence agents + */ +abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends + HasCustomTileLinkMessageTypes with + HasClientSideCoherencePolicy with + HasManagerSideCoherencePolicy -class ClientMetadata extends CoherenceMetadata { - val state = UInt(width = co.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata().asInstanceOf[this.type] -} -object ClientMetadata { - def apply(state: UInt) = { - val meta = new ClientMetadata - meta.state := state - meta - } -} - -class ManagerMetadata extends CoherenceMetadata { - val state = UInt(width = co.masterStateWidth) - val sharers = UInt(width = co.dir.width) - override def clone = new ManagerMetadata().asInstanceOf[this.type] -} -object ManagerMetadata { - def apply(state: UInt, sharers: UInt) = { - val meta = new ManagerMetadata - meta.state := state - meta.sharers := sharers - meta - } - def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) -} - -// This class encapsulates transformations on different directory information -// storage formats -abstract class DirectoryRepresentation(val width: Int) { - def pop(prev: UInt, id: UInt): UInt - def push(prev: UInt, id: UInt): UInt - def flush: UInt - def none(s: UInt): Bool - def one(s: UInt): Bool - def count(s: UInt): UInt - def next(s: UInt): UInt - def full(s: UInt): UInt -} - -class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { - def pop(prev: UInt, id: UInt) = UInt(0) - def push(prev: UInt, id: UInt) = UInt(0) - def flush = UInt(0) - def none(s: UInt) = Bool(false) - def one(s: UInt) = Bool(false) - def count(s: UInt) = UInt(nClients) - def next(s: UInt) = UInt(0) - def full(s: UInt) = SInt(-1, width = nClients).toUInt -} - -class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { - def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) - def push(prev: UInt, id: UInt) = prev | UIntToOH(id) - def flush = UInt(0, width = width) - def none(s: UInt) = s === UInt(0) - def one(s: UInt) = PopCount(s) === UInt(1) - def count(s: UInt) = PopCount(s) - def next(s: UInt) = PriorityEncoder(s) - def full(s: UInt) = s -} - -// Coherence policy inferface for clients and managers -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { - def nClientStates: Int - def nManagerStates: Int - def nAcquireTypes: Int - def nProbeTypes: Int - def nReleaseTypes: Int - def nGrantTypes: Int - def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nManagerStates) +/** This API defines the custom, coherence-policy-defined message types, + * as opposed to the built-in ones found in tilelink.scala. + * Policies must enumerate the custom messages to be sent over each + * channel, as well as which of them have associated data. + */ +trait HasCustomTileLinkMessageTypes { + val nAcquireTypes: Int def acquireTypeWidth = log2Up(nAcquireTypes) + val nProbeTypes: Int def probeTypeWidth = log2Up(nProbeTypes) + val nReleaseTypes: Int def releaseTypeWidth = log2Up(nReleaseTypes) + val nGrantTypes: Int def grantTypeWidth = log2Up(nGrantTypes) - val clientStatesWithReadPermission: Vec[UInt] - val clientStatesWithWritePermission: Vec[UInt] - val clientStatesWithDirtyData: Vec[UInt] val acquireTypesWithData = Nil // Only built-in Acquire types have data for now val releaseTypesWithData: Vec[UInt] val grantTypesWithData: Vec[UInt] +} +/** This API contains all functions required for client coherence agents. + * Policies must enumerate the number of client states and define their + * permissions with respect to memory operations. Policies must fill in functions + * to control which messages are sent and how metadata is updated in response + * to coherence events. These funtions are generally called from within the + * ClientMetadata class in metadata.scala + */ +trait HasClientSideCoherencePolicy { + // Client coherence states and their permissions + val nClientStates: Int + def clientStateWidth = log2Ceil(nClientStates) + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + + // Transaction initiation logic def isValid(meta: ClientMetadata): Bool - def isValid(meta: ManagerMetadata): Bool - def isHit(cmd: UInt, meta: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), clientStatesWithWritePermission.contains(meta.state), clientStatesWithReadPermission.contains(meta.state)) } - //TODO: Use outer protocol's clientState instead, remove this function: - def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + //TODO: Assumes all states with write permissions also have read permissions + def requiresAcquireOnSecondaryMiss( + first_cmd: UInt, + second_cmd: UInt, + meta: ClientMetadata): Bool = { + isWriteIntent(second_cmd) && + !isWriteIntent(first_cmd) || + (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: - def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = clientStatesWithDirtyData.contains(meta.state) - def needsWriteback(meta: ClientMetadata): Bool = - needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + // Determine which custom message type to use + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + + // Mutate ClientMetadata based on messages or cmds + def clientMetadataOnReset: ClientMetadata def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnFlush: ClientMetadata - - def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnFlush: ManagerMetadata - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt - def getReleaseType(p: Probe, meta: ClientMetadata): UInt - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt - - def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata } -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 2 - def nManagerStates = 2 - def nAcquireTypes = 1 - def nProbeTypes = 2 - def nReleaseTypes = 4 - def nGrantTypes = 1 +/** This API contains all functions required for manager coherence agents. + * Policies must enumerate the number of manager states. Policies must fill + * in functions to control which Probe and Grant messages are sent and how + * metadata should be updated in response to coherence events. These funtions + * are generally called from within the ManagerMetadata class in metadata.scala + */ +trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { + val nManagerStates: Int + def masterStateWidth = log2Ceil(nManagerStates) - val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Transaction probing logic + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + + // Determine which custom message type to use in response + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + + // Mutate ManagerMetadata based on messages or cmds + def managerMetadataOnReset: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached + meta.sharers, + dir.push(meta.sharers, dst))) + //state = meta.state) TODO: Fix 0-width wires in Chisel +} + +/** The following concrete implementations of CoherencePolicy each provide the + * functionality of one particular protocol. + */ + +/** A simple protocol with only two Client states. + * Data is always assumed to be dirty. + * Only a single client may ever have a copy of a block at a time. + */ +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 1 + val nProbeTypes = 2 + val nReleaseTypes = 4 + val nGrantTypes = 1 val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientValid) - val clientStatesWithWritePermission = Vec(clientValid) - val clientStatesWithDirtyData = Vec(clientValid) val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 2 + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (outstanding.a_type != acquireExclusive) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata(Mux(incoming.p_type === probeInvalidate, - clientInvalid, meta.state)) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) @@ -213,112 +159,85 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A simple protocol with only three Client states. + * Data is marked as dirty when written. + * Only a single client may ever have a copy of a block at a time. + */ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 1 - - val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 1 val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def isValid (meta: ClientMetadata) = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -333,75 +252,113 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), Bool(true)))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A protocol with only three Client states. + * Data is always assumed to be dirty. + * Multiple clients may share read permissions on a block at the same time. + */ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantShared, grantExclusive) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid(meta: ManagerMetadata) = meta.state != managerInvalid + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -409,9 +366,9 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -424,30 +381,82 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) } +} - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) +/** A protocol with four Client states. + * Data is marked as dirty when written. + * Multiple clients may share read permissions on a block at the same time. + */ +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + // Client states and functions + val nClientStates = 4 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -462,80 +471,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) - - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = - MuxLookup(cmd, probeCopy, Array( - M_FLUSH -> probeInvalidate, - M_PRODUCE -> probeDowngrade)) - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) - - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) -} - -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 4 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - - val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) - - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -544,13 +484,12 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { clientShared, meta.state), M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, - grantExclusive -> Mux(outstanding.a_type === acquireExclusive, - clientExclusiveDirty, clientExclusiveClean), + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty)))) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = @@ -560,188 +499,80 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) - MuxLookup(cmd, releaseCopyAck, Array( - M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), - M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), - M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) - } - - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = - MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 7 - def nManagerStates = 2 - def nAcquireTypes = 3 - def nProbeTypes = 4 - def nReleaseTypes = 10 - def nGrantTypes = 4 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 3 + val nProbeTypes = 4 + val nReleaseTypes = 10 + val nGrantTypes = 4 val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + // Client states and functions + val nClientStates = 7 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), - meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), - clientShared, meta.state), - M_CLEAN -> MuxLookup(meta.state, meta.state, Array( - clientExclusiveDirty -> clientExclusiveClean, - clientMigratoryDirty -> clientMigratoryClean))))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantShared -> clientShared, - grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireExclusive -> clientExclusiveDirty, - acquireShared -> clientExclusiveClean)), - grantExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireExclusive -> clientMigratoryDirty, - acquireShared -> clientMigratoryClean)))))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> meta.state, - probeDowngrade -> MuxLookup(meta.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid))))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next, - r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next)) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), - Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), - outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -769,43 +600,89 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(dirty, with_data, without_data) } - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - MuxLookup(a.a_type, grantShared, Array( - acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - acquireExclusive -> grantExclusive, - acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers))) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped, + incoming.is(releaseInvalidateDataMigratory) -> popped, + incoming.is(releaseInvalidateAckMigratory) -> popped)) + } } diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/directory.scala new file mode 100644 index 00000000..db555ad3 --- /dev/null +++ b/uncore/src/main/scala/directory.scala @@ -0,0 +1,43 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// This class encapsulates transformations on different directory information +// storage formats +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt + def full(s: UInt): UInt +} + +abstract trait HasDirectoryRepresentation { + val dir: DirectoryRepresentation +} + +class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(nClients) + def next(s: UInt) = UInt(0) + def full(s: UInt) = SInt(-1, width = nClients).toUInt +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s +} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5e5053be..4d5e1135 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) val dataBeats = params(TLDataBeats) - val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) val offsetBits = params(HTIFOffsetBits) @@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWriteBlock( + PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), - UncachedReadBlock(addr_block = init_addr)) + GetBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid @@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e58dd08d..f10b3257 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } +//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary + + //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemIO } - val co = params(TLCoherence) val mifTagBits = params(MIFTagBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) @@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + require(params(TLClientXactIdBits) <= params(MIFTagBits)) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + io.tl.grant <> gnt_arb.io.out + + val acq_has_data = io.tl.acquire.bits.payload.hasData() + val rel_has_data = io.tl.release.bits.payload.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - - io.uncached.acquire.ready := Bool(false) - io.uncached.grant.valid := Bool(false) - io.mem.resp.ready := Bool(false) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - - //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are - //acceptable Acquire types - val acq_has_data = io.uncached.acquire.bits.payload.hasData() - val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tl_done_out = Reg(init=Bool(false)) - val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + val grant_for_rel = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = tag_out, + manager_xact_id = UInt(0)) + val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_out(0), + a_type = tag_out >> UInt(1), + client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) - when(!active_out){ - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr_block - has_data := acq_has_data - tl_done_out := tl_wrap_out - mif_done_out := Bool(false) - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.tl.release.bits.payload.client_xact_id + addr_out := io.tl.release.bits.payload.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + addr_out := io.tl.acquire.bits.payload.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + } } - } - when(active_out) { - when(!cmd_sent_out) { - mem_cmd_q.io.enq.valid := Bool(true) - } - when(has_data) { - when(!tl_done_out) { - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out } } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) } } - when(mem_cmd_q.io.enq.fire()) { - cmd_sent_out := Bool(true) + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tag_out := io.mem.req_cmd.bits.tag + addr_out := io.mem.req_data.bits.data + has_data := io.mem.req_cmd.bits.rw + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + io.mem.req_cmd.bits.rw := rel_has_data + io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + io.mem.req_cmd.bits.rw := acq_has_data + io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + } + } } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || mif_done_out)) { - active_out := Bool(false) + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } } } - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - // Aggregate incoming MemIO responses into TL Grants - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) val active_in = Reg(init=Bool(false)) - val mif_done_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) - val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - - when(active_in) { - io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_in(0), + a_type = tag_in >> UInt(1), + client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { io.mem.resp.ready := Bool(true) when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data } } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits.payload := + ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = io.mem.resp.bits.tag(0), + a_type = io.mem.resp.bits.tag >> UInt(1), + client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } - - io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), - g_type = Grant.uncachedReadBlock, - client_xact_id = tag_in, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemPipeIO } - val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val a = Module(new MemIOTileLinkIOConverter(1)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) - a.io.uncached <> io.uncached + a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala new file mode 100644 index 00000000..8f5d7e90 --- /dev/null +++ b/uncore/src/main/scala/metadata.scala @@ -0,0 +1,188 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// Classes to represent coherence information in clients and managers +abstract class CoherenceMetadata extends Bundle { + val co = params(TLCoherencePolicy) + val id = params(TLId) +} + +/* The ClientMetadata stores the client-side coherence information, + such as permissions on the data and whether the data is dirty. + Its API can be used to make TileLink messages in response to + memory operations or TileLink Probes. +*/ +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + + def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + + def isValid(dummy: Int = 0): Bool = co.isValid(this) + def isHit(cmd: UInt): Bool = co.isHit(cmd, this) + def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) + def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) + def requiresReleaseOnCacheControl(cmd: UInt): Bool = + co.requiresReleaseOnCacheControl(cmd: UInt, this) + def requiresVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresReleaseOnCacheControl(M_FLUSH, this) + + def makeAcquire( + client_xact_id: UInt, + addr_block: UInt, + op_code: UInt): Acquire = { + Bundle(Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true))), + { case TLId => id }) + } + + def makeVoluntaryWriteback( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def makeRelease( + prb: Probe, + client_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(false), + r_type = co.getReleaseType(prb, this), + client_xact_id = client_xact_id, + addr_block = prb.addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onGrant(incoming: Grant, pending: UInt): ClientMetadata = + Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + def onProbe(incoming: Probe): ClientMetadata = + Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + def onHit(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) + def onCacheControl(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) +} + +object ClientMetadata { + def apply(state: UInt) = { + val meta = new ClientMetadata + meta.state := state + meta + } + def onReset = new ClientMetadata().co.clientMetadataOnReset +} + +/* The ManagerMetadata stores manager-side information about the status + of a cache block, including whether it has any known sharers. Its + API can be used to create Probe and Grant TileLink messages. +*/ +class ManagerMetadata extends CoherenceMetadata { + // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + val sharers = UInt(width = co.dir.width) + + def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel + this.sharers === rhs.sharers + def full(dummy: Int = 0) = co.dir.full(this.sharers) + + def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresProbes(M_FLUSH, this) + + def makeProbe(cmd: UInt, addr_block: UInt): Probe = + Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + + def makeProbe(acq: Acquire): Probe = + Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + + def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = + makeProbe(M_FLUSH, addr_block) + + def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + Bundle(Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = rel.client_xact_id, + manager_xact_id = manager_xact_id), { case TLId => id }) + } + + def makeGrant( + acq: Acquire, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Bundle(Grant( + is_builtin_type = acq.isBuiltInType(), + g_type = Mux(acq.isBuiltInType(), + acq.getBuiltInGrantType(), + co.getGrantType(acq, this)), + client_xact_id = acq.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onRelease(incoming: Release, src: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + + def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) +} + +object ManagerMetadata { + def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + val meta = new ManagerMetadata + //meta.state := state TODO: Fix 0-width wires in Chisel + meta.sharers := sharers + meta + } + def apply() = { + val meta = new ManagerMetadata + //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel + meta.sharers := meta.co.dir.flush + meta + } + def onReset = new ManagerMetadata().co.managerMetadataOnReset +} + +/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + that is a Manager with respect to some inner caches and a Client with + respect to some outer cache. +*/ +class HierarchicalMetadata extends CoherenceMetadata { + val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) + val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) + def ===(rhs: HierarchicalMetadata): Bool = + this.inner === rhs.inner && this.outer === rhs.outer + def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) +} + +object HierarchicalMetadata { + def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + val m = new HierarchicalMetadata + m.inner := inner + m.outer := outer + m + } + def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) +} + +case object InnerTLId extends Field[String] +case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index afbf4dc0..c8996ddf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -8,30 +8,35 @@ import scala.math.max // external requirements or design space exploration // case object TLId extends Field[String] // Unique name per network -case object TLCoherence extends Field[CoherencePolicy] -case object TLAddrBits extends Field[Int] +case object TLCoherencePolicy extends Field[CoherencePolicy] +case object TLBlockAddrBits extends Field[Int] case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] +case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) + val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = M_SZ - val tlUncachedOperandSizeBits = MT_SZ - val tlSubblockUnionBits = max(tlWriteMaskBits, + val tlMemoryOpcodeBits = M_SZ + val tlMemoryOperandSizeBits = MT_SZ + val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes), + tlCoh.acquireTypeWidth) + val tlAcquireUnionBits = max(tlWriteMaskBits, (tlByteAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 - val co = params(TLCoherence) - val networkPreservesPointToPointOrdering = false //TODO: check physical network type + tlMemoryOperandSizeBits + + tlMemoryOpcodeBits)) + 1 + val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), + tlCoh.grantTypeWidth) + 1 + val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) - def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block - def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr + def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block + def conflicts(addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { @@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -abstract trait HasTileLinkData extends HasTileLinkBeatId { +trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } @@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val builtin_type = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) - val subblock = Bits(width = tlSubblockUnionBits) + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) + val union = Bits(width = tlAcquireUnionBits) // Utility funcs for accessing subblock union - val opSizeOff = tlByteAddrBits + 1 - val opCodeOff = tlUncachedOperandSizeBits + opSizeOff - val opMSB = tlAtomicOpcodeBits + opCodeOff - def allocate(dummy: Int = 0) = subblock(0) - def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) - def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) - def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) - def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + val opCodeOff = 1 + val opSizeOff = tlMemoryOpcodeBits + opCodeOff + val addrByteOff = tlMemoryOperandSizeBits + opSizeOff + val addrByteMSB = tlByteAddrBits + addrByteOff + def allocate(dummy: Int = 0) = union(0) + def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) + def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + + // Assumes no custom types have data + def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType - def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = - Probe(co.getProbeType(this, meta), this.addr_block) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { - Grant( - builtin_type = this.builtin_type, - g_type = co.getGrantType(this, meta), - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id, - addr_beat = addr_beat, - data = data - ) + def getBuiltInGrantType(dummy: Int = 0): UInt = { + MuxLookup(this.a_type, Grant.ackType, Array( + Acquire.getType -> Grant.dataBeatType, + Acquire.getBlockType -> Grant.dataBlockType, + Acquire.putType -> Grant.ackType, + Acquire.putBlockType -> Grant.ackType, + Acquire.putAtomicType -> Grant.dataBeatType)) } } object Acquire { - val nBuiltinAcquireTypes = 5 + val nBuiltInTypes = 5 //TODO: Use Enum - def uncachedRead = UInt(0) - def uncachedReadBlock = UInt(1) - def uncachedWrite = UInt(2) - def uncachedWriteBlock = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) - def typesWithMultibeatData = Vec(uncachedWriteBlock) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock - def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock + def getType = UInt("b000") + def getBlockType = UInt("b001") + def putType = UInt("b010") + def putBlockType = UInt("b011") + def putAtomicType = UInt("b100") + def typesWithData = Vec(putType, putBlockType, putAtomicType) + def typesWithMultibeatData = Vec(putBlockType) + def typesOnSubBlocks = Vec(putType, getType, putAtomicType) def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt // Most generic constructor def apply( - builtin_type: Bool, + is_builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0), - subblock: UInt = UInt(0)): Acquire = { + union: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.builtin_type := builtin_type + acq.is_builtin_type := is_builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block acq.addr_beat := addr_beat acq.data := data - acq.subblock := subblock + acq.union := union acq } - // For cached types - def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { - apply( - builtin_type = Bool(false), - a_type = a_type, - client_xact_id = client_xact_id, - addr_block = addr_block) - } // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -177,58 +169,73 @@ object Acquire { } // Asks for a single TileLink beat of data -object UncachedRead { +object Get { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedRead, + is_builtin_type = Bool(true), + a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - subblock = alloc) + union = Cat(M_XRD, alloc)) } } // Asks for an entire cache block of data -object UncachedReadBlock { +object GetBlock { def apply( client_xact_id: UInt = UInt(0), addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedReadBlock, + is_builtin_type = Bool(true), + a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - subblock = alloc.toUInt) + union = Cat(M_XRD, alloc)) } } -object UncachedWrite { +// Writes up to a single TileLink beat of data, using mask +object Put { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask, - alloc: Bool = Bool(true)): Acquire = { + write_mask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWrite, + is_builtin_type = Bool(true), + a_type = Acquire.putType, addr_block = addr_block, addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - subblock = Cat(write_mask, alloc)) + union = Cat(write_mask, Bool(true))) } } -// For full block of data -object UncachedWriteBlock { +// Writes an entire cache block of data +object PutBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + } def apply( client_xact_id: UInt, addr_block: UInt, @@ -236,17 +243,18 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWriteBlock, + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(Acquire.fullWriteMask, alloc)) + union = Cat(Acquire.fullWriteMask, alloc)) } } -object UncachedAtomic { +// Performs an atomic operation in the outer memory +object PutAtomic { def apply( client_xact_id: UInt, addr_block: UInt, @@ -256,48 +264,30 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedAtomic, + is_builtin_type = Bool(true), + a_type = Acquire.putAtomicType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) + union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) } } class Probe extends ManagerToClientChannel with HasCacheBlockAddress { - val p_type = UInt(width = co.probeTypeWidth) + val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t - def makeRelease( - client_xact_id: UInt, - meta: ClientMetadata = co.clientMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(false), - r_type = co.getReleaseType(this, meta), - client_xact_id = client_xact_id, - addr_block = this.addr_block, - addr_beat = addr_beat, - data = data) - } } object Probe { - val co = new Probe().co def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } - - def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { - apply(co.getProbeType(M_FLUSH, meta), addr_block) - } } @@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) + val r_type = UInt(width = tlCoh.releaseTypeWidth) val voluntary = Bool() // Helper funcs def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) //TODO: Assumes all releases write back full cache blocks: - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary - def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { - Grant( - g_type = Grant.voluntaryAck, - builtin_type = Bool(true), // Grant.voluntaryAck is built-in type - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id - ) - } + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } object Release { - val co = new Release().co def apply( voluntary: Bool, r_type: UInt, @@ -346,68 +324,52 @@ object Release { rel.voluntary := voluntary rel } - - def makeVoluntaryWriteback( - meta: ClientMetadata, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(true), - r_type = co.getReleaseType(M_FLUSH, meta), - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = addr_beat, - data = data) - } } class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val builtin_type = Bool() - val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) // Helper funcs - def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(builtin_type, + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), - co.grantTypesWithData.contains(g_type)) + tlCoh.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(builtin_type, + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), Grant.typesWithMultibeatData.contains(g_type), - co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) - def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = new Finish + val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) f.manager_xact_id := this.manager_xact_id f } } object Grant { - val nBuiltinGrantTypes = 5 - //TODO Use Enum - def voluntaryAck = UInt(0) - def uncachedRead = UInt(1) - def uncachedReadBlock = UInt(2) - def uncachedWrite = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) - def typesWithMultibeatData= Vec(uncachedReadBlock) + val nBuiltInTypes = 4 + def voluntaryAckType = UInt("b00") + def ackType = UInt("b01") + def dataBeatType = UInt("b10") + def dataBlockType = UInt("b11") + def typesWithData = Vec(dataBlockType, dataBeatType) + def typesWithMultibeatData= Vec(dataBlockType) def apply( - builtin_type: Bool, + is_builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.builtin_type := builtin_type + gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -415,23 +377,12 @@ object Grant { gnt.data := data gnt } - - def getGrantTypeForUncached(a: Acquire): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedWriteBlock -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class Finish extends ClientToManagerChannel with HasManagerTransactionId - // Complete IO definitions for two types of TileLink clients -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) @@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](utl: UncachedTileLinkIO) = { + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { val conv = Module(new TileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: TileLinkIO) = tl } abstract trait HasArbiterTypes { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 1230771c..3c334376 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,437 +2,147 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] case object NIncoherentClients extends Field[Int] case object NCoherentClients extends Field[Int] -case object L2StoreDataQueueDepth extends Field[Int] -case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] -case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object L2CoherencePolicy extends Field[CoherencePolicy] -abstract trait CoherenceAgentParameters extends UsesParameters - with TileLinkParameters { +trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nCoherentClients = params(NCoherentClients) val nIncoherentClients = params(NIncoherentClients) val nClients = nCoherentClients + nIncoherentClients - val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) - val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases + def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerDataBeats = outerTLParams(TLDataBeats) + val outerDataBits = outerTLParams(TLDataBits) + def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerDataBeats = innerTLParams(TLDataBeats) + val innerDataBits = innerTLParams(TLDataBits) + val innerBeatAddrBits = log2Up(innerDataBeats) + val innerByteAddrBits = log2Up(innerDataBits/8) + require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } +abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters +abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters -class DataQueueLocation extends Bundle with CoherenceAgentParameters { - val idx = UInt(width = dqIdxBits) - val loc = UInt(width = log2Ceil(nDataQueueLocations)) -} -object DataQueueLocation { - def apply(idx: UInt, loc: UInt) = { - val d = new DataQueueLocation - d.idx := idx - d.loc := loc - d +trait HasCoherenceAgentWiringHelpers { + def doOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( + out: DecoupledIO[S], + ins: Seq[DecoupledIO[S]]) { + def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter( + out.bits.clone, + ins.size, + out.bits.payload.tlDataBeats, + lock _)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } + + def doInputRouting[T <: HasManagerTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { + val idx = in.bits.payload.manager_xact_id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + in.ready := Vec(outs.map(_.ready)).read(idx) } } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } +trait HasInnerTLIO extends CoherenceAgentBundle { + val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val incoherent = Vec.fill(nClients){Bool()}.asInput + def iacq(dummy: Int = 0) = inner.acquire.bits.payload + def iprb(dummy: Int = 0) = inner.probe.bits.payload + def irel(dummy: Int = 0) = inner.release.bits.payload + def ignt(dummy: Int = 0) = inner.grant.bits.payload + def ifin(dummy: Int = 0) = inner.finish.bits.payload } -class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) { - - val internalDataBits = new DataQueueLocation().getWidth - val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - - // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Queue to store impending UncachedWrite data - val acquire = io.inner.acquire - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val) - val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } - - // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - - // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() - val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width - val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } - - // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Wire probe requests to clients - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() - val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - - // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) - val is_in_sdq = outer_data_ptr.loc === inStoreQueue - val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && - outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( - inStoreQueue -> sdq(outer_data_ptr.idx), - inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer <> outer_arb.io.out - - // Update SDQ valid bits - when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | - PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) - } +trait HasUncachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } - -abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) - val nClients = params(NCoherentClients) + params(NIncoherentClients) - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cfin = io.inner.finish.bits - val macq = io.outer.acquire.bits - val mgnt = io.outer.grant.bits - +trait HasCachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new TileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def oprb(dummy: Int = 0) = outer.probe.bits.payload + def orel(dummy: Int = 0) = outer.release.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) +class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_inner_data = Reg(init=Bool(false)) - // TODO: assert that all releases have full blocks of data - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) - - io.has_acquire_conflict := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - - when(collect_inner_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(inner_data_cnt) := crel.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_inner_data := crel.payload.hasMultibeatData() - state := Mux(crel.payload.hasData(), s_outer, - Mux(crel.payload.requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } +abstract class CoherenceAgent extends CoherenceAgentModule { + def innerTL: TileLinkIO + def outerTL: TileLinkIO + def incoherent: Vec[Bool] } -class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) +abstract class ManagerCoherenceAgent extends CoherenceAgent + with HasCoherenceAgentWiringHelpers { + val io = new ManagerTLIO + def innerTL = io.inner + def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def incoherent = io.incoherent +} - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) +class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO - val collect_inner_data = Reg(init=Bool(false)) - //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) +abstract class HierarchicalCoherenceAgent extends CoherenceAgent { + val io = new HierarchicalTLIO + def innerTL = io.inner + def outerTL = io.outer + def incoherent = io.incoherent +} +trait HasTrackerConflictIO extends Bundle { + val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) +} - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) +class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO +class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO - val pending_outer_write = xact.hasData() - val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - - io.has_acquire_conflict := xact.conflicts(cacq.payload) && - !collect_inner_data && - (state != s_idle) - io.has_release_match := xact.conflicts(crel.payload) && - !crel.payload.isVoluntary() && - (state != s_idle) - - val outer_write_acq = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = crel.payload.addr_beat, - data = crel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe() - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - when(collect_inner_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := cacq.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } +abstract class XactTracker extends CoherenceAgentModule { + def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = cacq.payload.hasData() - val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) - when(io.inner.acquire.valid) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_inner_data := cacq.payload.hasMultibeatData() - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } - } - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { - when(crel.payload.hasData()) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - when(outer_data_done) { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } .otherwise { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { // Read data from outer memory (possibly what was just written) - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { state := s_mem_resp } - } - is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - io.outer.acquire.bits.payload := outer_write_acq - when(outer_data_done) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_mem_resp) { // Wait to forward grants from outer memory - when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(cgnt_data_done) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { // Wait for transaction to complete - when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { - state := s_idle - } - } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + beat: UInt = UInt(0)) = { + connectDataBeatCounter(in.fire(), in.bits.payload, beat) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } }