Fix BroadcastHub AcquiteTracker allocation bug and clean up tracker wiring

uncore/src/main/scala/broadcast.scala

@@ -40,53 +40,53 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent
   trackerList.map(_.io.incoherent := io.incoherent.toBits)
 
   // Queue to store impending Put data
-  val acquire = io.inner.acquire
+  val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) }
   val sdq_val = Reg(init=Bits(0, sdqDepth))
   val sdq_alloc_id = PriorityEncoder(~sdq_val)
   val sdq_rdy = !sdq_val.andR
-  val sdq_enq = acquire.fire() && acquire.bits.payload.hasData()
+  val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData()
-  val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) }
+  when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data }
-  when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data }
 
   // Handle acquire transaction initiation
-  val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
-  val block_acquires = any_acquire_conflict
   val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
-  for( i <- 0 until trackerList.size ) {
+  val trackerAcquireIOs =  trackerList.map(_.io.inner.acquire)
-    val t = trackerList(i).io.inner
+  val acquireMatchList = trackerList.map(_.io.has_acquire_match)
-    alloc_arb.io.in(i).valid := t.acquire.ready
+  val any_acquire_matches = acquireMatchList.reduce(_||_)
-    t.acquire.bits := acquire.bits
+  val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b}
-    t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
+  val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b}
-    t.acquire.valid := alloc_arb.io.in(i).ready
+  val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx)
+  trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach {
+    case((tracker, arb), i) =>
+      arb.valid := tracker.ready
+      tracker.bits := io.inner.acquire.bits
+      tracker.bits.payload.data :=
+        DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
+      tracker.valid := arb.ready && (acquire_idx === UInt(i))
   }
-  acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
+  val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
-  alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
+  io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) &&
+                              sdq_rdy && !block_acquires
+  alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires
 
   // Queue to store impending Voluntary Release data
-  val release = io.inner.release
+  val voluntary = io.irel().isVoluntary()
-  val voluntary = release.bits.payload.isVoluntary()
+  val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData()
-  val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData()
   val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
-  val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 
+  val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 
-  when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data }
+  when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data }
 
   // Handle releases, which might be voluntary and might have data
   val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b}
-  for( i <- 0 until trackerList.size ) {
+  val trackerReleaseIOs = trackerList.map(_.io.inner.release)
-    val t = trackerList(i).io.inner
+  trackerReleaseIOs.zipWithIndex.foreach {
-    t.release.bits := release.bits 
+    case(tracker, i) =>
-    t.release.bits.payload.data := (if (i < nReleaseTransactors) 
+      tracker.bits := io.inner.release.bits
-                                      DataQueueLocation(rel_data_cnt, inVolWBQueue)
+      tracker.bits.payload.data := DataQueueLocation(rel_data_cnt,
-                                    else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits
+                                     (if(i < nReleaseTransactors) inVolWBQueue
-    t.release.valid := release.valid && (release_idx === UInt(i))
+                                      else inClientReleaseQueue)).toBits
+      tracker.valid := io.inner.release.valid && (release_idx === UInt(i))
   }
-  release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx)
+  io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx)
-
-  // Wire finished transaction acks
-  val ack = io.inner.finish
-  trackerList.map(_.io.inner.finish.valid := ack.valid)
-  trackerList.map(_.io.inner.finish.bits := ack.bits)
-  ack.ready := Bool(true)
 
   // Wire probe requests and grant reply to clients, finish acks from clients
   // Note that we bypass the Grant data subbundles
@@ -107,7 +107,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent
   val free_sdq = io.outer.acquire.fire() &&
                   io.outer.acquire.bits.payload.hasData() &&
                   outer_data_ptr.loc === inStoreQueue
-  io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array(
+  io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array(
                                           inStoreQueue -> sdq(outer_data_ptr.idx),
                                           inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
   io.outer <> outer_arb.io.out
@@ -216,8 +216,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact
   val coh = ManagerMetadata.onReset
 
   assert(!(state != s_idle && xact.isBuiltInType() && 
-      Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)),
+      Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType,
-    "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO
+        Acquire.prefetchType).contains(xact.a_type)),
+    "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO
 
   val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1)))
   val pending_probes = Reg(init=Bits(0, width = nCoherentClients))
@@ -238,17 +239,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact
   val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant)
   val pending_ognt_ack = Reg(init=Bool(false))
   val pending_outer_write = xact.hasData()
-  val pending_outer_read = io.ignt().hasData()
   val pending_outer_write_ = io.iacq().hasData()
+  val pending_outer_read = io.ignt().hasData()
   val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData()
 
   io.has_acquire_conflict := xact.conflicts(io.iacq()) && 
-                              !collect_iacq_data &&
+                              (state != s_idle) &&
-                              (state != s_idle)
+                              !collect_iacq_data
-  io.has_release_match := xact.conflicts(io.irel()) && 
+  io.has_acquire_match := xact.conflicts(io.iacq()) &&
-                              !io.irel().isVoluntary() &&
+                              collect_iacq_data
-                              (state != s_idle)
+  io.has_release_match := xact.conflicts(io.irel()) &&
-  io.has_acquire_match := Bool(false)
+                            !io.irel().isVoluntary() &&
+                            (state === s_probe)
 
   val outer_write_acq = Bundle(PutBlock(
                                 client_xact_id = UInt(trackerId),
@@ -282,6 +284,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact
   io.inner.release.ready := Bool(false)
   io.inner.finish.ready := Bool(false)
 
+  assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() &&
+    io.inner.acquire.bits.header.src != xact_src),
+    "AcquireTracker accepted data beat from different network source than initial request.")
+
+  assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() &&
+    io.iacq().client_xact_id != xact.client_xact_id),
+    "AcquireTracker accepted data beat from different client transaction than initial request.")
+
+  assert(!(state === s_idle && io.inner.acquire.fire() &&
+    io.iacq().addr_beat != UInt(0)),
+    "AcquireTracker initialized with a tail data beat.")
+
   when(collect_iacq_data) {
     io.inner.acquire.ready := Bool(true)
     when(io.inner.acquire.valid) {

uncore/src/main/scala/cache.scala

@@ -356,33 +356,34 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule
   (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits)
 
   // Handle acquire transaction initiation
-  val acquire = io.inner.acquire
   val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
-  val acquireList =  trackerList.map(_.io.inner.acquire)
+  val trackerAcquireIOs =  trackerList.map(_.io.inner.acquire)
   val acquireMatchList = trackerList.map(_.io.has_acquire_match)
   val any_acquire_matches = acquireMatchList.reduce(_||_)
   val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b}
   val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b}
   val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx)
-  acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) =>
+  trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { 
-    arb.valid := acq.ready
+    case((tracker, arb), i) =>
-    acq.bits := acquire.bits
+      arb.valid := tracker.ready
-    acq.valid := arb.ready && (acquire_idx === UInt(i))
+      tracker.bits := io.inner.acquire.bits
+      tracker.valid := arb.ready && (acquire_idx === UInt(i))
   }
   val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
-  acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires
+  io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) &&
-  alloc_arb.io.out.ready := acquire.valid && !block_acquires
+                              !block_acquires
+  alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires
 
   // Wire releases from clients
-  val release = io.inner.release
   val release_idx = Vec(trackerList.map(_.io.has_release_match) :+
                           wb.io.has_release_match).indexWhere{b: Bool => b}
-  val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release
+  val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release
-  releaseList.zipWithIndex.map { case(r, i) =>
+  trackerReleaseIOs.zipWithIndex.foreach { 
-    r.bits := release.bits
+    case(tracker, i) =>
-    r.valid := release.valid && (release_idx === UInt(i))
+      tracker.bits := io.inner.release.bits
+      tracker.valid := io.inner.release.valid && (release_idx === UInt(i))
   }
-  release.ready := Vec(releaseList.map(_.ready)).read(release_idx)
+  io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx)
 
   // Wire probe requests and grant reply to clients, finish acks from clients
   doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe)
@@ -658,8 +659,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker {
                               !collect_iacq_data
   io.has_acquire_match := xact.conflicts(io.iacq()) &&
                               collect_iacq_data
-  io.has_release_match := !io.irel().isVoluntary() &&
+  io.has_release_match := xact.conflicts(io.irel()) &&
-                            (xact.addr_block === io.irel().addr_block) &&
+                            !io.irel().isVoluntary() &&
                             (state === s_probe)
 
   // If we're allocating in this cache, we can use the current metadata
@@ -729,6 +730,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker {
   io.wb.req.bits.way_en := xact_way_en
   io.wb.req.bits.id := UInt(trackerId)
 
+  assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() &&
+    io.inner.acquire.bits.header.src != xact_src),
+    "AcquireTracker accepted data beat from different network source than initial request.")
+
+  assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() &&
+    io.iacq().client_xact_id != xact.client_xact_id),
+    "AcquireTracker accepted data beat from different client transaction than initial request.")
+
+  assert(!(state === s_idle && io.inner.acquire.fire() &&
+    io.iacq().addr_beat != UInt(0)),
+    "AcquireTracker initialized with a tail data beat.")
+
   when(collect_iacq_data) {
     io.inner.acquire.ready := Bool(true)
     when(io.inner.acquire.valid) {