refactor ready/valid logic for routing release messages in the l2

uncore/src/main/scala/broadcast.scala

@@ -83,19 +83,15 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p)
   // Handle releases, which might be voluntary and might have data
   val trackerReleaseIOs = trackerList.map(_.io.inner.release)
   val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits
-  val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits
+  io.inner.release.ready := releaseReadys.orR
-  val release_idx = PriorityEncoder(releaseMatches)
-  io.inner.release.ready := releaseReadys(release_idx)
   trackerReleaseIOs.zipWithIndex.foreach {
     case(tracker, i) =>
-      tracker.valid := io.inner.release.valid && (release_idx === UInt(i))
+      tracker.valid := io.inner.release.valid
       tracker.bits := io.inner.release.bits
       tracker.bits.data := DataQueueLocation(rel_data_cnt,
                                      (if(i < nReleaseTransactors) inVolWBQueue
                                       else inClientReleaseQueue)).toBits
   }
-  assert(!(io.inner.release.valid && !releaseMatches.orR),
-    "Non-voluntary release should always have a Tracker waiting for it.")
 
   // Wire probe requests and grant reply to clients, finish acks from clients
   // Note that we bypass the Grant data subbundles
@@ -129,85 +125,68 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p)
 
 class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) {
   val io = new ManagerXactTrackerIO
+  pinAllReadyValidLow(io)
 }
 
 class BroadcastVoluntaryReleaseTracker(trackerId: Int)
                                       (implicit p: Parameters) extends BroadcastXactTracker()(p) {
-  val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4)
+  val s_idle :: s_busy :: Nil = Enum(UInt(), 2)
   val state = Reg(init=s_idle)
 
   val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId })))
   val coh = ManagerMetadata.onReset
 
-  val collect_irel_data = Reg(init=Bool(false))
+  val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats))
-  val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats))
+  val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats))
-  val irel_data_done = connectIncomingDataBeatCounter(io.inner.release)
+  val pending_ignt = Reg(init=Bool(false))
-  val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire)
 
-  io.has_acquire_conflict := Bool(false)
+  val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt)
-  io.has_release_match := io.irel().isVoluntary()
-  io.has_acquire_match := Bool(false)
 
-  io.outer.acquire.valid := Bool(false)
+  // Accept a voluntary Release (and any further beats of data)
-  io.outer.grant.ready := Bool(false)
+  pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release))
-  io.inner.acquire.ready := Bool(false)
+  io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR
-  io.inner.probe.valid := Bool(false)
+  when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data }
-  io.inner.release.ready := Bool(false)
-  io.inner.grant.valid := Bool(false)
-  io.inner.finish.ready := Bool(false)
-
-  io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId))
 
+  // Write the voluntarily written back data to outer memory using an Acquire.PutBlock
   //TODO: Use io.outer.release instead?
+  pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) |
+                      addPendingBitWhenBeatHasData(io.inner.release)
+  val curr_write_beat = PriorityEncoder(pending_writes)
+  io.outer.acquire.valid := state === s_busy && pending_writes.orR
   io.outer.acquire.bits := PutBlock( 
                                client_xact_id = UInt(trackerId),
                                addr_block = xact.addr_block,
-                               addr_beat = oacq_data_cnt,
+                               addr_beat = curr_write_beat,
-                               data = xact.data_buffer(oacq_data_cnt))
+                               data = xact.data_buffer(curr_write_beat))
                              (p.alterPartial({ case TLId => outerTLId }))
 
-  when(collect_irel_data) {
+  // Send an acknowledgement
-    io.inner.release.ready := Bool(true)
+  io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid
-    when(io.inner.release.valid) {
+  io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId))
-      xact.data_buffer(io.irel().addr_beat) := io.irel().data
+  when(io.inner.grant.fire()) { pending_ignt := Bool(false) }
-      irel_data_valid := irel_data_valid.bitSet(io.irel().addr_beat, Bool(true))
+  io.outer.grant.ready := state === s_busy && io.inner.grant.ready
-    }
-    when(irel_data_done) { collect_irel_data := Bool(false) }
-  }
 
-  switch (state) {
+  // State machine updates and transaction handler metadata intialization
-    is(s_idle) {
+  when(state === s_idle && io.inner.release.fire()) {
-      io.inner.release.ready := Bool(true)
-      when( io.inner.release.valid ) {
     xact := io.irel()
-        xact.data_buffer(UInt(0)) := io.irel().data
+    when(io.irel().hasMultibeatData()) {
-        collect_irel_data := io.irel().hasMultibeatData()
+      pending_irels := dropPendingBitWhenBeatHasData(io.inner.release)
-        irel_data_valid := io.irel().hasData() << io.irel().addr_beat
+    }. otherwise { 
-        state := Mux(io.irel().hasData(), s_outer,
+      pending_irels := UInt(0)
-                   Mux(io.irel().requiresAck(), s_ack, s_idle))
-      }
-    }
-    is(s_outer) {
-      io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt)
-      when(oacq_data_done) { 
-        state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ?
-      }
-    }
-    is(s_grant) { // Forward the Grant.voluntaryAck
-      io.outer.grant.ready := io.inner.grant.ready
-      io.inner.grant.valid := io.outer.grant.valid 
-      when(io.inner.grant.fire()) {
-        state := Mux(io.ignt().requiresAck(), s_ack, s_idle)
-      }
-    }
-    is(s_ack) {
-      // TODO: This state is unnecessary if no client will ever issue the
-      // pending Acquire that caused this writeback until it receives the 
-      // Grant.voluntaryAck for this writeback
-      io.inner.finish.ready := Bool(true)
-      when(io.inner.finish.valid) { state := s_idle }
     }
+    pending_writes := addPendingBitWhenBeatHasData(io.inner.release)
+    pending_ignt := io.irel().requiresAck()
+    state := s_busy
   }
+  when(state === s_busy && all_pending_done) { state := s_idle }
+
+  // These IOs are used for routing in the parent
+  io.has_acquire_match := Bool(false)
+  io.has_acquire_conflict := Bool(false)
+
+  // Checks for illegal behavior
+  assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()),
+    "VoluntaryReleaseTracker accepted Release that wasn't voluntary!")
 }
 
 class BroadcastAcquireTracker(trackerId: Int)
@@ -247,9 +226,6 @@ class BroadcastAcquireTracker(trackerId: Int)
                               !collect_iacq_data
   io.has_acquire_match := xact.conflicts(io.iacq()) &&
                               collect_iacq_data
-  io.has_release_match := xact.conflicts(io.irel()) &&
-                            !io.irel().isVoluntary() &&
-                            (state === s_probe)
 
   val outerParams = p.alterPartial({ case TLId => outerTLId })
 
@@ -357,8 +333,9 @@ class BroadcastAcquireTracker(trackerId: Int)
       }
 
       // Handle releases, which may have data to be written back
-      io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready
+      val matches = xact.conflicts(io.irel()) && !io.irel().isVoluntary()
-      when(io.inner.release.valid) {
+      io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches
+      when(io.inner.release.valid && matches) {
         when(io.irel().hasData()) {
           io.outer.acquire.valid := Bool(true)
           when(io.outer.acquire.ready) {

uncore/src/main/scala/cache.scala

@@ -405,15 +405,14 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p)
 
   // Wire releases from clients
   val releaseReadys = Vec(trackerAndWbIOs.map(_.inner.release.ready)).toBits
-  val releaseMatches = Vec(trackerAndWbIOs.map(_.has_release_match)).toBits
+  io.inner.release.ready := releaseReadys.orR
-  io.inner.release.ready := (releaseMatches & releaseReadys).orR
   trackerAndWbIOs foreach { tracker =>
     tracker.inner.release.bits := io.inner.release.bits
-    tracker.inner.release.valid := io.inner.release.valid && tracker.has_release_match
+    tracker.inner.release.valid := io.inner.release.valid
   }
-  assert(PopCount(releaseMatches) <= UInt(nReleaseTransactors),
+  assert(PopCount(releaseReadys) <= UInt(nReleaseTransactors),
     "At most a single tracker should match for any given Release")
-  assert(!(io.inner.release.valid && !releaseMatches.orR),
+  assert(!(io.inner.release.valid && !releaseReadys.orR),
     "Non-voluntary release should always have a Tracker waiting for it.")
 
   // Wire probe requests and grant reply to clients, finish acks from clients
@@ -491,19 +490,6 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p)
     val isPartial = a.wmask() =/= Acquire.fullWriteMask
     addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a)
   }
-
-  def pinAllReadyValidLow[T <: Data](b: Bundle) {
-    b.elements.foreach {
-      _._2 match {
-        case d: DecoupledIO[_] =>
-          if(d.ready.dir == OUTPUT) d.ready := Bool(false)
-          else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
-        case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) 
-        case b: Bundle => pinAllReadyValidLow(b)
-        case _ =>
-      }
-    }
-  }
 }
 
 class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) {
@@ -528,7 +514,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends
 
   // Accept a voluntary Release (and any further beats of data)
   pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release))
-  io.inner.release.ready := state === s_idle || pending_irels.orR
+  io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR
   when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data }
 
   // Begin a transaction by getting the current block metadata
@@ -566,7 +552,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends
                                          xact_old_meta.coh.outer)
 
   // State machine updates and transaction handler metadata intialization
-  when(state === s_idle && io.inner.release.valid) {
+  when(state === s_idle && io.inner.release.fire()) {
     xact := io.irel()
     when(io.irel().hasMultibeatData()) {
       pending_irels := dropPendingBitWhenBeatHasData(io.inner.release)
@@ -587,7 +573,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends
   when(state === s_meta_write && io.meta.write.ready) { state := s_idle }
 
   // These IOs are used for routing in the parent
-  io.has_release_match := io.irel().isVoluntary()
   io.has_acquire_match := Bool(false)
   io.has_acquire_conflict := Bool(false)
 
@@ -816,7 +801,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra
 
   // Handle incoming releases from clients, which may reduce sharer counts
   // and/or write back dirty data
-  io.inner.release.ready := state === s_inner_probe
+  io.inner.release.ready := state === s_inner_probe && 
+                              io.irel().conflicts(xact_addr_block) &&
+                              !io.irel().isVoluntary()
   val pending_coh_on_irel = HierarchicalMetadata(
                               pending_coh.inner.onRelease(io.irel()), // Drop sharer
                               Mux(io.irel().hasData(),     // Dirty writeback
@@ -1025,9 +1012,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra
 
   // These IOs are used for routing in the parent
   val in_same_set = xact_addr_idx === io.iacq().addr_block(idxMSB,idxLSB)
-  io.has_release_match := io.irel().conflicts(xact_addr_block) &&
-                          !io.irel().isVoluntary() &&
-                          io.inner.release.ready
   io.has_acquire_match := iacq_can_merge || iacq_same_xact
   io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match
   //TODO: relax from in_same_set to xact.conflicts(io.iacq())?
@@ -1083,7 +1067,9 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac
   // and/or write back dirty data
   val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel())
   val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR)
-  io.inner.release.ready := state === s_inner_probe || state === s_busy
+  io.inner.release.ready := (state === s_inner_probe || state === s_busy) &&
+                              io.irel().conflicts(xact_addr_block) && 
+                              !io.irel().isVoluntary()
   when(io.inner.release.fire()) {
     xact.coh.inner := inner_coh_on_irel
     data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data
@@ -1150,7 +1136,6 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac
   when(state === s_wb_resp ) { state := s_idle }
 
   // These IOs are used for routing in the parent
-  io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready
   io.has_acquire_match := Bool(false)
   io.has_acquire_conflict := Bool(false)
 }

uncore/src/main/scala/uncore.scala

@@ -117,7 +117,6 @@ abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends Cohere
 trait HasTrackerConflictIO extends Bundle {
   val has_acquire_conflict = Bool(OUTPUT)
   val has_acquire_match = Bool(OUTPUT)
-  val has_release_match = Bool(OUTPUT)
 }
 
 class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p)
@@ -154,4 +153,17 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule(
 
   def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt =
     ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id)
+
+  def pinAllReadyValidLow[T <: Data](b: Bundle) {
+    b.elements.foreach {
+      _._2 match {
+        case d: DecoupledIO[_] =>
+          if(d.ready.dir == OUTPUT) d.ready := Bool(false)
+          else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
+        case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) 
+        case b: Bundle => pinAllReadyValidLow(b)
+        case _ =>
+      }
+    }
+  }
 }