diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 4b6627aa..c3eb7038 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -83,19 +83,15 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) // Handle releases, which might be voluntary and might have data val trackerReleaseIOs = trackerList.map(_.io.inner.release) val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits - val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits - val release_idx = PriorityEncoder(releaseMatches) - io.inner.release.ready := releaseReadys(release_idx) + io.inner.release.ready := releaseReadys.orR trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) + tracker.valid := io.inner.release.valid tracker.bits := io.inner.release.bits tracker.bits.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits } - assert(!(io.inner.release.valid && !releaseMatches.orR), - "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -129,85 +125,68 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { val io = new ManagerXactTrackerIO + pinAllReadyValidLow(io) } class BroadcastVoluntaryReleaseTracker(trackerId: Int) (implicit p: Parameters) extends BroadcastXactTracker()(p) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val s_idle :: s_busy :: Nil = Enum(UInt(), 2) val state = Reg(init=s_idle) val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val coh = ManagerMetadata.onReset - val collect_irel_data = Reg(init=Bool(false)) - val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) - io.has_acquire_conflict := Bool(false) - io.has_release_match := io.irel().isVoluntary() - io.has_acquire_match := Bool(false) + val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt) - io.outer.acquire.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR + when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } + // Write the voluntarily written back data to outer memory using an Acquire.PutBlock //TODO: Use io.outer.release instead? + pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.outer.acquire.valid := state === s_busy && pending_writes.orR io.outer.acquire.bits := PutBlock( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = xact.data_buffer(oacq_data_cnt)) + addr_beat = curr_write_beat, + data = xact.data_buffer(curr_write_beat)) (p.alterPartial({ case TLId => outerTLId })) - when(collect_irel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact.data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid := irel_data_valid.bitSet(io.irel().addr_beat, Bool(true)) - } - when(irel_data_done) { collect_irel_data := Bool(false) } - } + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + io.outer.grant.ready := state === s_busy && io.inner.grant.ready - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := io.irel() - xact.data_buffer(UInt(0)) := io.irel().data - collect_irel_data := io.irel().hasMultibeatData() - irel_data_valid := io.irel().hasData() << io.irel().addr_beat - state := Mux(io.irel().hasData(), s_outer, - Mux(io.irel().requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) - when(oacq_data_done) { - state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? - } - } - is(s_grant) { // Forward the Grant.voluntaryAck - io.outer.grant.ready := io.inner.grant.ready - io.inner.grant.valid := io.outer.grant.valid - when(io.inner.grant.fire()) { - state := Mux(io.ignt().requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.fire()) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_busy } + when(state === s_busy && all_pending_done) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // Checks for illegal behavior + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } class BroadcastAcquireTracker(trackerId: Int) @@ -247,9 +226,6 @@ class BroadcastAcquireTracker(trackerId: Int) !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_probe) val outerParams = p.alterPartial({ case TLId => outerTLId }) @@ -357,8 +333,9 @@ class BroadcastAcquireTracker(trackerId: Int) } // Handle releases, which may have data to be written back - io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { + val matches = xact.conflicts(io.irel()) && !io.irel().isVoluntary() + io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches + when(io.inner.release.valid && matches) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3192cb66..e01187eb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -405,15 +405,14 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) // Wire releases from clients val releaseReadys = Vec(trackerAndWbIOs.map(_.inner.release.ready)).toBits - val releaseMatches = Vec(trackerAndWbIOs.map(_.has_release_match)).toBits - io.inner.release.ready := (releaseMatches & releaseReadys).orR + io.inner.release.ready := releaseReadys.orR trackerAndWbIOs foreach { tracker => tracker.inner.release.bits := io.inner.release.bits - tracker.inner.release.valid := io.inner.release.valid && tracker.has_release_match + tracker.inner.release.valid := io.inner.release.valid } - assert(PopCount(releaseMatches) <= UInt(nReleaseTransactors), + assert(PopCount(releaseReadys) <= UInt(nReleaseTransactors), "At most a single tracker should match for any given Release") - assert(!(io.inner.release.valid && !releaseMatches.orR), + assert(!(io.inner.release.valid && !releaseReadys.orR), "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients @@ -491,19 +490,6 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) val isPartial = a.wmask() =/= Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } - - def pinAllReadyValidLow[T <: Data](b: Bundle) { - b.elements.foreach { - _._2 match { - case d: DecoupledIO[_] => - if(d.ready.dir == OUTPUT) d.ready := Bool(false) - else if(d.valid.dir == OUTPUT) d.valid := Bool(false) - case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) - case b: Bundle => pinAllReadyValidLow(b) - case _ => - } - } - } } class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { @@ -528,7 +514,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends // Accept a voluntary Release (and any further beats of data) pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) - io.inner.release.ready := state === s_idle || pending_irels.orR + io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } // Begin a transaction by getting the current block metadata @@ -566,7 +552,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends xact_old_meta.coh.outer) // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.valid) { + when(state === s_idle && io.inner.release.fire()) { xact := io.irel() when(io.irel().hasMultibeatData()) { pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) @@ -587,7 +573,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends when(state === s_meta_write && io.meta.write.ready) { state := s_idle } // These IOs are used for routing in the parent - io.has_release_match := io.irel().isVoluntary() io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) @@ -816,7 +801,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data - io.inner.release.ready := state === s_inner_probe + io.inner.release.ready := state === s_inner_probe && + io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() val pending_coh_on_irel = HierarchicalMetadata( pending_coh.inner.onRelease(io.irel()), // Drop sharer Mux(io.irel().hasData(), // Dirty writeback @@ -1025,9 +1012,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // These IOs are used for routing in the parent val in_same_set = xact_addr_idx === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() && - io.inner.release.ready io.has_acquire_match := iacq_can_merge || iacq_same_xact io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? @@ -1083,7 +1067,9 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac // and/or write back dirty data val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) - io.inner.release.ready := state === s_inner_probe || state === s_busy + io.inner.release.ready := (state === s_inner_probe || state === s_busy) && + io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() when(io.inner.release.fire()) { xact.coh.inner := inner_coh_on_irel data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data @@ -1150,7 +1136,6 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac when(state === s_wb_resp ) { state := s_idle } // These IOs are used for routing in the parent - io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 34405aae..c90adeeb 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -117,7 +117,6 @@ abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends Cohere trait HasTrackerConflictIO extends Bundle { val has_acquire_conflict = Bool(OUTPUT) val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) } class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) @@ -154,4 +153,17 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[_] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } }