1R/W L2 data array?

Add TLDataBeats to new LLC; all bmarks pass

uncore/src/main/scala/cache.scala

@@ -134,6 +134,7 @@ class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata)
     with HasL2Id {
   override def clone = new L2MetaWriteReq().asInstanceOf[this.type]
 }
+
 class L2MetaResp extends L2HellaCacheBundle
   with HasL2Id 
   with HasL2InternalRequestState
@@ -156,10 +157,10 @@ class L2MetadataArray extends L2HellaCacheModule {
   meta.io.read <> io.read
   meta.io.write <> io.write
   
-  val s1_clk_en = Reg(next = io.read.fire())
   val s1_tag = RegEnable(io.read.bits.tag, io.read.valid)
   val s1_id = RegEnable(io.read.bits.id, io.read.valid)
   def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f))
+  val s1_clk_en = Reg(next = io.read.fire())
   val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag)
   val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits
   val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en)
@@ -220,11 +221,11 @@ class L2DataArray extends L2HellaCacheModule {
     }
     array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid))
   }
-  io.resp.valid := ShiftRegister(io.read.valid, 2)
+  io.resp.valid := ShiftRegister(io.read.fire(), 1)
-  io.resp.bits.id := ShiftRegister(io.read.bits.id, 2)
+  io.resp.bits.id := ShiftRegister(io.read.bits.id, 1)
-  io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp)
+  io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp)
 
-  io.read.ready := Bool(true)
+  io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W?
   io.write.ready := Bool(true)
 }
 
@@ -233,7 +234,6 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends
 
   require(isPow2(nSets))
   require(isPow2(nWays)) 
-  require(refillCycles == 1)
 
   val tshrfile = Module(new TSHRFile(bankId, innerId, outerId))
   val meta = Module(new L2MetadataArray)
@@ -257,8 +257,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   }
 
   // Wiring helper funcs
-  def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) {
+  def doOutputArbitration[T <: Data](out: DecoupledIO[T], 
-    val arb = Module(new RRArbiter(out.bits.clone, ins.size))
+                                      ins: Seq[DecoupledIO[T]], 
+                                      count: Int = 1, 
+                                      lock: T => Bool = (a: T) => Bool(true)) {
+    val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock))
     out <> arb.io.out
     arb.io.in zip ins map { case (a, in) => a <> in }
   }
@@ -282,7 +285,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   val acquire = io.inner.acquire
   val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
   val block_acquires = any_acquire_conflict
-
   val alloc_arb = Module(new Arbiter(Bool(), trackerList.size))
   for( i <- 0 until trackerList.size ) {
     val t = trackerList(i).io.inner
@@ -293,9 +295,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires
   alloc_arb.io.out.ready := acquire.valid && !block_acquires
 
-  // Handle probe requests
-  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
-
   // Handle releases, which might be voluntary and might have data
   val release = io.inner.release
   val voluntary = co.isVoluntary(release.bits.payload)
@@ -310,10 +309,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   }
   release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases
 
-  // Reply to initial requestor
+  // Wire finished transaction acks
-  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
-
-  // Free finished transactions on ack
   val finish = io.inner.finish
   val finish_idx = finish.bits.payload.master_xact_id
   trackerList.zipWithIndex.map { case (t, i) => 
@@ -322,20 +318,26 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   trackerList.map(_.io.inner.finish.bits := finish.bits)
   finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx)
 
-  // Arbitrate for the outer memory port
+  // Wire probe requests to clients
+  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
+
+  // Wire grant reply to initiating client
+  def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload)
+  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _)
+
+  // Create an arbiter for the one memory port
   val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
                          {case TLId => outerId})
   outer_arb.io.in zip  trackerList map { case(arb, t) => arb <> t.io.outer }
   io.outer <> outer_arb.io.out
 
-  // Local memory
+  // Wire local memories
   doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read))
   doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write))
   doOutputArbitration(io.data.read, trackerList.map(_.io.data.read))
   doOutputArbitration(io.data.write, trackerList.map(_.io.data.write))
   doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp))
   doInputRouting(io.data.resp, trackerList.map(_.io.data.resp))
-
 }
 
 
@@ -356,21 +358,34 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa
   val c_ack = io.inner.finish.bits
   val m_gnt = io.outer.grant.bits
 
-  def mergeData(acq: Acquire, data: UInt): UInt = {
+  def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = {
-    //TODO wite mask
+    //TODO apply acq's write mask
-    Mux(co.messageHasData(acq), acq.data, data)
+    Mux(co.messageHasData(acq), old_data, new_data)
   }
 }
 
 class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
   val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7)
   val state = Reg(init=s_idle)
-  val xact  = Reg{ new Release }
+
-  val xact_internal = Reg{ new L2MetaResp }
+  val xact_src = Reg(io.inner.release.bits.header.src.clone)
-  val init_client_id = Reg(init=UInt(0, width = log2Up(nClients)))
+  val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
+  val xact_addr = Reg(io.inner.release.bits.payload.addr.clone)
+  val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
+  val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
+  val xact_tag_match = Reg{ Bool() }
+  val xact_meta = Reg{ new L2Metadata }
+  val xact_way_en = Reg{ Bits(width = nWays) }
+  val xact = Release(xact_r_type, xact_addr, xact_client_xact_id)
+
+  val collect_inner_data = Reg(init=Bool(false))
+  val (inner_data_cnt, inner_data_done) =
+    Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
+  val (local_data_cnt, local_data_done) =
+    Counter(io.data.write.fire(), tlDataBeats)
 
   io.has_acquire_conflict := Bool(false)
-  io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && 
+  io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && 
                                (state != s_idle)
 
   io.outer.grant.ready := Bool(false)
@@ -382,38 +397,50 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
   io.inner.finish.ready := Bool(false)
 
   io.inner.grant.bits.header.src := UInt(bankId)
-  io.inner.grant.bits.header.dst := init_client_id
+  io.inner.grant.bits.header.dst := xact_src
   io.inner.grant.bits.payload := Grant(Bool(false),
-                                        co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh),
+                                        co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh),
-                                        xact.client_xact_id,
+                                        xact_client_xact_id,
                                         UInt(trackerId))
 
   io.data.read.valid := Bool(false)
   io.data.write.valid := Bool(false)
   io.data.write.bits.id := UInt(trackerId)
-  io.data.write.bits.way_en := xact_internal.way_en
+  io.data.write.bits.way_en := xact_way_en
-  io.data.write.bits.addr := xact.addr
+  io.data.write.bits.addr := Cat(xact_addr, local_data_cnt)
   io.data.write.bits.wmask := SInt(-1)
-  io.data.write.bits.data := xact.data
+  io.data.write.bits.data := xact_data(local_data_cnt)
   io.meta.read.valid := Bool(false)
   io.meta.read.bits.id := UInt(trackerId)
-  io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits)
+  io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits)
-  io.meta.read.bits.tag := xact.addr >> UInt(untagBits)
+  io.meta.read.bits.tag := xact_addr >> UInt(untagBits)
   io.meta.write.valid := Bool(false)
   io.meta.write.bits.id := UInt(trackerId)
-  io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits)
+  io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits)
-  io.meta.write.bits.way_en := xact_internal.way_en
+  io.meta.write.bits.way_en := xact_way_en
-  io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits)
+  io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits)
   io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, 
-                                                            xact_internal.meta.coh, 
+                                                            xact_meta.coh, 
-                                                            init_client_id)
+                                                            xact_src)
+
+  when(collect_inner_data) {
+    io.inner.release.ready := Bool(true)
+    when(io.inner.release.valid) {
+      xact_data(inner_data_cnt) := c_rel.payload.data
+    }
+    when(inner_data_done) { collect_inner_data := Bool(false) }
+  }
 
   switch (state) {
     is(s_idle) {
       io.inner.release.ready := Bool(true)
       when( io.inner.release.valid ) {
-        xact := c_rel.payload
+        xact_src := c_rel.header.src
-        init_client_id := c_rel.header.src
+        xact_r_type := c_rel.payload.r_type
+        xact_addr := c_rel.payload.addr
+        xact_client_xact_id := c_rel.payload.client_xact_id
+        xact_data(UInt(0)) := c_rel.payload.data
+        collect_inner_data := co.messageHasData(c_rel.payload)
         state := s_meta_read
       }
     }
@@ -423,15 +450,18 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
     }
     is(s_meta_resp) {
       when(io.meta.resp.valid) {
-        xact_internal := io.meta.resp.bits
+        xact_tag_match := io.meta.resp.bits.tag_match
+        xact_meta := io.meta.resp.bits.meta
+        xact_way_en := io.meta.resp.bits.way_en
         state := Mux(io.meta.resp.bits.tag_match, 
                    Mux(co.messageHasData(xact), s_data_write, s_meta_write),
                    s_grant)
       }
     }
     is(s_data_write) {
-      io.data.write.valid := Bool(true)
+      io.data.write.valid := (if(tlDataBeats == 1) Bool(true) 
-      when(io.data.write.ready) { state := s_meta_write }
+                                  else !collect_inner_data || (local_data_cnt < inner_data_cnt))
+      when(local_data_done) { state := s_meta_write }
     }
     is(s_meta_write) {
       io.meta.write.valid := Bool(true)
@@ -454,72 +484,100 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
 class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) {
   val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16)
   val state = Reg(init=s_idle)
-  val xact  = Reg{ new Acquire }
+
-  val xact_internal = Reg{ new L2MetaResp }
+  val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
-  val init_client_id = Reg(init = UInt(0, width = log2Up(nClients)))
+  val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone)
+  val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
+  val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
+  val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
+  val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
+  val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
+  val xact_tag_match = Reg{ Bool() }
+  val xact_meta = Reg{ new L2Metadata }
+  val xact_way_en = Reg{ Bits(width = nWays) }
+  val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock)
+
   val crel_had_data = Reg(init = Bool(false))
   val crel_was_voluntary = Reg(init = Bool(false))
   val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients)))
   val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth))
-  val wb_buffer = Reg{xact.data.clone}
+  val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
-  val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits))
+  val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits))
+
+  val collect_cacq_data = Reg(init=Bool(false))
+  //TODO: zero width wires
+  val (cacq_data_cnt, cacq_data_done) = 
+    Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats)
+  val (crel_data_cnt, crel_data_done) = 
+    Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
+  val (cgnt_data_cnt, cgnt_data_done) = 
+    Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats)
+  val (outer_data_write_cnt, outer_data_write_done) = 
+    Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
+  val (outer_data_resp_cnt, outer_data_resp_done) = 
+    Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats)
+  val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats)
+  val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats)
+  val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats)
 
   val release_count = Reg(init = UInt(0, width = log2Up(nClients+1)))
   val pending_probes = Reg(init = co.dir().flush)
   val curr_p_id = co.dir().next(pending_probes)
   
-  val is_uncached = xact.uncached
+  val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh)
-  val tag_match = xact_internal.tag_match
+  val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh)
-  val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh)
+  val needs_probes = co.requiresProbes(xact, xact_meta.coh)
-  val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh)
+  //TODO: uncached does or does not allocate
-  val needs_probes = co.requiresProbes(xact, xact_internal.meta.coh)
-  //TODO: does allocate
 
-  io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && 
+  //TODO: Are there any races between lines with the same idx?
-                              (state != s_idle) //TODO: Also indexes
+  //TODO: Allow hit under miss for stores
+  io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) &&
+                              xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) &&
+                              (state != s_idle) &&
+                              !collect_cacq_data
   io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) ||
-                               co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && 
+                               co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) &&
-                               (state != s_idle) //TODO: Also indexes?
+                               (state != s_idle)
 
   val next_coh_on_release = co.masterMetadataOnRelease(
                                     c_rel.payload, 
-                                    xact_internal.meta.coh,
+                                    xact_meta.coh,
                                     c_rel.header.src)
   val next_coh_on_grant = co.masterMetadataOnGrant(
                                     c_gnt.payload, 
-                                    xact_internal.meta.coh,
+                                    xact_meta.coh,
                                     c_gnt.header.dst)
 
-  val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), 
+  val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), 
                           { case TLId => outerId })
-  val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer),
+  val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)),
                           { case TLId => outerId })
-  val outer_read = Bundle(UncachedRead( xact.addr, UInt(trackerId)), { case TLId => outerId })
+  val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId })
 
   io.outer.acquire.valid := Bool(false)
   io.outer.acquire.bits.payload := outer_read //default
   io.outer.acquire.bits.header.src := UInt(bankId)
   io.outer.grant.ready := Bool(true) //grant.data -> xact.data
 
-  val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr)
+  val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr)
-  val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, addr_wb)
+  val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr)
-  //TODO inner_probe_mprb
+  //TODO cprb_for_mprb
   io.inner.probe.valid := Bool(false)
   io.inner.probe.bits.header.src := UInt(bankId)
   io.inner.probe.bits.header.dst := curr_p_id
-  io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, 
+  io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, 
-                                      cprb_for_mwb,
+                                      cprb_for_mvwb,
                                       cprb_for_cacq)
 
-  val cgnt_for_cacq = Grant(xact.uncached, co.getGrantType(xact, xact_internal.meta.coh),
+  val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh),
-                              xact.client_xact_id,
+                              xact_client_xact_id,
                               UInt(trackerId),
-                              xact.data)
+                              xact_data(cgnt_data_cnt))
-  val cgnt_for_cwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0))
+  val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0))
   io.inner.grant.valid := Bool(false)
   io.inner.grant.bits.header.src := UInt(bankId)
-  io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id)
+  io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src)
-  io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq)
+  io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq)
 
   io.inner.acquire.ready := Bool(false)
   io.inner.release.ready := Bool(false)
@@ -527,31 +585,45 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
 
   io.data.read.valid := Bool(false)
   io.data.read.bits.id := UInt(trackerId)
-  io.data.read.bits.way_en := xact_internal.way_en
+  io.data.read.bits.way_en := xact_way_en
-  io.data.read.bits.addr := xact.addr 
+  io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt)
   io.data.write.valid := Bool(false)
   io.data.write.bits.id := UInt(trackerId)
-  io.data.write.bits.way_en := xact_internal.way_en
+  io.data.write.bits.way_en := xact_way_en
-  io.data.write.bits.addr := xact.addr
+  io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt)
   io.data.write.bits.wmask := SInt(-1)
-  io.data.write.bits.data := xact.data
+  io.data.write.bits.data := xact_data(local_data_write_cnt)
   io.meta.read.valid := Bool(false)
   io.meta.read.bits.id := UInt(trackerId)
-  io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits)
+  io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits)
-  io.meta.read.bits.tag := xact.addr >> UInt(untagBits)
+  io.meta.read.bits.tag := xact_addr >> UInt(untagBits)
   io.meta.write.valid := Bool(false)
   io.meta.write.bits.id := UInt(trackerId)
-  io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits)
+  io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits)
-  io.meta.write.bits.way_en := xact_internal.way_en
+  io.meta.write.bits.way_en := xact_way_en
-  io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits)
+  io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits)
   io.meta.write.bits.data.coh := next_coh_on_grant
 
+  when(collect_cacq_data) {
+    io.inner.acquire.ready := Bool(true)
+    when(io.inner.acquire.valid) {
+      xact_data(cacq_data_cnt) := c_acq.payload.data
+    }
+    when(cacq_data_done) { collect_cacq_data := Bool(false) }
+  }
+
   switch (state) {
     is(s_idle) {
       io.inner.acquire.ready := Bool(true)
       when( io.inner.acquire.valid ) {
-        xact := c_acq.payload
+        xact_uncached := c_acq.payload.uncached
-        init_client_id := c_acq.header.src
+        xact_a_type := c_acq.payload.a_type
+        xact_addr := c_acq.payload.addr
+        xact_client_xact_id := c_acq.payload.client_xact_id
+        xact_data(UInt(0)) := c_acq.payload.data
+        xact_subblock := c_acq.payload.subblock
+        xact_src := c_acq.header.src
+        collect_cacq_data := co.messageHasData(c_acq.payload)
         state := s_meta_read
       }
     }
@@ -561,21 +633,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
     }
     is(s_meta_resp) {
       when(io.meta.resp.valid) {
+        xact_tag_match := io.meta.resp.bits.tag_match
+        xact_meta := io.meta.resp.bits.meta
+        xact_way_en := io.meta.resp.bits.way_en
         val coh = io.meta.resp.bits.meta.coh
         val _tag_match = io.meta.resp.bits.tag_match
         val _needs_writeback = !_tag_match && co.needsWriteback(coh)
         val _is_hit = _tag_match && co.isHit(xact, coh)
         val _needs_probes = co.requiresProbes(xact, coh)
-        xact_internal := io.meta.resp.bits
         when(_needs_probes) {
           val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent
           val mask_self = mask_incoherent & 
-                            ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << init_client_id)
+                            ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src)
           pending_probes := mask_self
           release_count := co.dir().count(mask_self)
           crel_had_data := Bool(false)
           crel_was_voluntary := Bool(false)
-        }  
+        } 
         state := Mux(_tag_match,
                   Mux(_is_hit,
                     Mux(_needs_probes, s_probe, s_data_read_hit),
@@ -594,25 +668,41 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
       // Handle releases, which may have data being written back
       io.inner.release.ready := Bool(true)
       when(io.inner.release.valid) {
-        xact_internal.meta.coh := next_coh_on_release
+        xact_meta.coh := next_coh_on_release
+
+        // Handle released dirty data
         when(co.messageHasData(c_rel.payload)) {
           crel_had_data := Bool(true)
-          when(tag_match) {
+          when(xact_tag_match) { // Hit, so merge new write with released data
-            xact.data := mergeData(xact, io.inner.release.bits.payload.data)
+            //TODO make sure cacq data is actually present before merging
-          } .otherwise {
+            xact_data(crel_data_cnt) := mergeData(xact,
-            wb_buffer := io.inner.release.bits.payload.data
+                                                  xact_data(crel_data_cnt),
+                                                  io.inner.release.bits.payload.data)
+          } .otherwise {         // Miss, we're voluntarily evicting this data
+            wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data
           }
         }
+
+        // Voluntary releases don't count against the release countdown
+        // because we will get a further release ack from that client in
+        // response to our probe. We don't send a grant acknowledging
+        // a writeback or decrement release_count until we've received 
+        // all the data beats.
         when(co.isVoluntary(c_rel.payload)) {
-          crel_was_voluntary := Bool(true)
+          when(!co.messageHasData(c_rel.payload) || crel_data_done) {
-          crel_wb_src := c_rel.header.src
+            crel_was_voluntary := Bool(true)
-          crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh)
+            crel_wb_src := c_rel.header.src
-        }
+            crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh)
-        when(!co.isVoluntary(c_rel.payload)) {
+          }
-          release_count := release_count - UInt(1)
+        } .otherwise {
+          when(!co.messageHasData(c_rel.payload) || crel_data_done) {
+            release_count := release_count - UInt(1)
+          }
         }
       }
 
+      // If we saw a voluntary writeback, we need to send an extra grant
+      // to acknowledge it.
       when(crel_was_voluntary) { 
         io.inner.grant.valid := Bool(true)
         when(io.inner.grant.ready) { 
@@ -621,7 +711,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
       }
 
       when(release_count === UInt(0) && !crel_was_voluntary) {
-        state := Mux(tag_match,
+        state := Mux(xact_tag_match,
                   Mux(is_hit,
                     Mux(crel_had_data, s_data_write, s_data_read_hit),
                     s_outer_read),
@@ -630,20 +720,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
     }
     is(s_data_read_wb) {
       io.data.read.valid := Bool(true)
-      when(io.data.read.ready) {
+      when(local_data_read_done) { state := s_data_resp_wb }
-        state := s_data_resp_wb
+      when(io.data.resp.valid) {
+        wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data
       }
     }
     is(s_data_resp_wb) {
-      when(io.data.resp.valid) {
+      when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data }
-        wb_buffer := io.data.resp.bits.data
+      when(local_data_resp_done) { state := s_outer_write_wb }
-        state := s_outer_write_wb
-      }
     }
     is(s_outer_write_wb) {
       io.outer.acquire.valid := Bool(true)
       io.outer.acquire.bits.payload := outer_write_wb
-      when(io.outer.acquire.ready) { 
+      when(outer_data_write_done) { 
         state := s_outer_read
       }
     }
@@ -657,27 +746,35 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
     is(s_outer_resp) {
       io.outer.grant.ready := Bool(true)
       when(io.outer.grant.valid) {
-        xact.data := mergeData(xact, io.outer.grant.bits.payload.data)
+        //TODO make sure cacq data is actually present before merging
-        //TODO: set pending client state in xact_internal.meta.coh
+        xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), 
-        state := Mux(co.messageHasData(io.outer.grant.bits.payload),
+                                                      io.outer.grant.bits.payload.data)
-                      s_data_write, s_data_read_hit)
+        //TODO: set pending client state in xact_meta.coh
+        when(outer_data_resp_done) { 
+          state := Mux(co.messageHasData(io.outer.grant.bits.payload),
+                                          s_data_write, s_data_read_hit)
+        }
       }
     }
     is(s_data_read_hit) {
       io.data.read.valid := Bool(true)
-      when(io.data.read.ready) {
+      when(io.data.resp.valid) {
-        state := s_data_resp_hit
+        //TODO make sure cacq data is actually present before merging
+        xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), 
+                                                      io.data.resp.bits.data)
       }
+      when(local_data_read_done) { state := s_data_resp_hit }
     }
     is(s_data_resp_hit) {
       when(io.data.resp.valid) {
-        xact.data := mergeData(xact, io.data.resp.bits.data)
+        xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), 
-        state := s_meta_write
+                                                      io.data.resp.bits.data)
       }
+      when(local_data_resp_done) { state := s_meta_write }
     }
     is(s_data_write) {
       io.data.write.valid := Bool(true)
-      when(io.data.write.ready) {
+      when(local_data_write_done) {
         state := s_meta_write
       }
     }
@@ -687,7 +784,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
     }
     is(s_grant) {
       io.inner.grant.valid := Bool(true)
-      when(io.inner.grant.ready) { 
+      when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { 
         state := Mux(co.requiresAckForGrant(c_gnt.payload),
           s_busy, s_idle) 
       }

uncore/src/main/scala/coherence.scala

@@ -205,9 +205,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di
   def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = {
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
-    MuxBundle(uncached, Array(
+    Mux(g.uncached, uncached, cached)
-      g.is(grantReadExclusive) -> cached
-    ))
   }
 
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
@@ -339,9 +337,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d
   def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = {
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
-    MuxBundle(uncached, Array(
+    Mux(g.uncached, uncached, cached)
-      g.is(grantReadExclusive) -> cached
-    ))
   }
 
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
@@ -485,10 +481,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d
   def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = {
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
-    MuxBundle(uncached, Array(
+    Mux(g.uncached, uncached, cached)
-      g.is(grantReadShared) -> cached,
-      g.is(grantReadExclusive) -> cached
-    ))
   }
 
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
@@ -644,11 +637,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(
   def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = {
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
-    MuxBundle(uncached, Array(
+    Mux(g.uncached, uncached, cached)
-      g.is(grantReadShared) -> cached,
-      g.is(grantReadExclusive) -> cached,
-      g.is(grantReadExclusiveAck) -> cached
-    ))
   }
 
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
@@ -817,12 +806,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo
   def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = {
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
-    MuxBundle(uncached, Array(
+    Mux(g.uncached, uncached, cached)
-      g.is(grantReadShared) -> cached,
-      g.is(grantReadExclusive) -> cached,
-      g.is(grantReadExclusiveAck) -> cached,
-      g.is(grantReadMigratory) -> cached
-    ))
   }
 
 

uncore/src/main/scala/tilelink.scala

@@ -78,16 +78,19 @@ object Acquire {
   def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite
   def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite
 
-  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
+  def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = {
     val acq = new Acquire
-    acq.uncached := Bool(false)
+    acq.uncached := uncached
     acq.a_type := a_type
     acq.addr := addr
     acq.client_xact_id := client_xact_id
     acq.data := data
-    acq.subblock := UInt(0)
+    acq.subblock := subblock
     acq
   }
+  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
+    apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0))
+  }
   def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = {
     apply(a_type, addr, client_xact_id, UInt(0))
   }

uncore/src/main/scala/uncore.scala

@@ -15,20 +15,10 @@ abstract trait CoherenceAgentParameters extends UsesParameters
   val nTransactors = nReleaseTransactors + nAcquireTransactors
   val nClients = params(NClients)
   val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats
-  val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) +
+  val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
-                    log2Ceil(tlDataBeats)
   val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
 }
 
-abstract class CoherenceAgent(innerId: String, outerId: String) extends Module
-    with CoherenceAgentParameters {
-  val io = new Bundle {
-    val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip
-    val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId})
-    val incoherent = Vec.fill(nClients){Bool()}.asInput
-  }
-}
-
 class DataQueueLocation extends Bundle with CoherenceAgentParameters {
   val idx = UInt(width = dqIdxBits)
   val loc = UInt(width = log2Ceil(nDataQueueLocations))
@@ -42,6 +32,15 @@ object DataQueueLocation {
   }
 }
 
+abstract class CoherenceAgent(innerId: String, outerId: String) extends Module
+    with CoherenceAgentParameters {
+  val io = new Bundle {
+    val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip
+    val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId})
+    val incoherent = Vec.fill(nClients){Bool()}.asInput
+  }
+}
+
 class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends 
     CoherenceAgent(innerId, outerId) {
 
@@ -103,6 +102,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends
   }
   release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases
 
+  // Wire finished transaction acks
+  val ack = io.inner.finish
+  trackerList.map(_.io.inner.finish.valid := ack.valid)
+  trackerList.map(_.io.inner.finish.bits := ack.bits)
+  ack.ready := Bool(true)
+
   // Wire probe requests to clients
   val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size))
   io.inner.probe <> probe_arb.io.out
@@ -115,12 +120,6 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends
   io.inner.grant <> grant_arb.io.out
   grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant }
 
-  // Wire finished transaction acks
-  val ack = io.inner.finish
-  trackerList.map(_.io.inner.finish.valid := ack.valid)
-  trackerList.map(_.io.inner.finish.bits := ack.bits)
-  ack.ready := Bool(true)
-
   // Create an arbiter for the one memory port
   val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
                          { case TLId => outerId; case TLDataBits => internalDataBits })
@@ -164,9 +163,13 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module {
 class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
   val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4)
   val state = Reg(init=s_idle)
-  val xact  = Reg{ new Release }
+
-  val init_client_id = Reg(init=UInt(0, width = log2Up(nClients)))
+  val xact_src = Reg(io.inner.release.bits.header.src.clone)
-  val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
+  val xact_r_type = Reg(io.inner.release.bits.payload.r_type)
+  val xact_addr = Reg(io.inner.release.bits.payload.addr.clone)
+  val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone)
+  val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) }
+
   val collect_inner_data = Reg(init=Bool(false))
   val (inner_data_cnt, inner_data_done) =
     Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats)
@@ -174,33 +177,35 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
     Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
 
   io.has_acquire_conflict := Bool(false)
-  io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && 
+  io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && 
                                (state != s_idle)
 
   io.outer.grant.ready := Bool(false)
   io.outer.acquire.valid := Bool(false)
-  io.outer.acquire.bits.header.src := UInt(bankId) 
-  //io.outer.acquire.bits.header.dst TODO
-  io.outer.acquire.bits.payload := Bundle(UncachedWrite(
-                                            xact.addr,
-                                            UInt(trackerId),
-                                            data_ptrs(outer_data_cnt)),
-                                    { case TLId => outerId })
   io.inner.acquire.ready := Bool(false)
   io.inner.probe.valid := Bool(false)
   io.inner.release.ready := Bool(false)
   io.inner.grant.valid := Bool(false)
+  io.inner.finish.ready := Bool(false)
+
   io.inner.grant.bits.header.src := UInt(bankId)
-  io.inner.grant.bits.header.dst := init_client_id
+  io.inner.grant.bits.header.dst := xact_src
   io.inner.grant.bits.payload := Grant(Bool(false),
                                         co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush),
-                                        xact.client_xact_id,
+                                        xact_client_xact_id,
                                         UInt(trackerId))
 
+  io.outer.acquire.bits.header.src := UInt(bankId) 
+  io.outer.acquire.bits.payload := Bundle(UncachedWrite(
+                                            xact_addr,
+                                            UInt(trackerId),
+                                            xact_data(outer_data_cnt)),
+                                    { case TLId => outerId })
+
   when(collect_inner_data) {
     io.inner.release.ready := Bool(true)
     when(io.inner.release.valid) {
-      data_ptrs(inner_data_cnt) := c_rel.payload.data
+      xact_data(inner_data_cnt) := c_rel.payload.data
     }
     when(inner_data_done) { collect_inner_data := Bool(false) }
   }
@@ -209,15 +214,18 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
     is(s_idle) {
       io.inner.release.ready := Bool(true)
       when( io.inner.release.valid ) {
-        xact := c_rel.payload
+        xact_src := c_rel.header.src
-        init_client_id := c_rel.header.src
+        xact_r_type := c_rel.payload.r_type
-        data_ptrs(UInt(0)) := c_rel.payload.data
+        xact_addr := c_rel.payload.addr
+        xact_client_xact_id := c_rel.payload.client_xact_id
+        xact_data(UInt(0)) := c_rel.payload.data
         collect_inner_data := co.messageHasData(c_rel.payload)
         state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack)
       }
     }
     is(s_outer) {
-      io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
+      io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) 
+                                  else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
       when(outer_data_done) { state := s_ack }
     }
     is(s_ack) {
@@ -230,28 +238,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
 class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) {
   val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6)
   val state = Reg(init=s_idle)
-  val xact  = Reg{ new Acquire }
+
-  val init_client_id = Reg(init=UInt(0, width = log2Up(nClients)))
+  val xact_src = Reg(io.inner.acquire.bits.header.src.clone)
-  //TODO: Will need id reg for merged release xacts
+  val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone)
-  val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
+  val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone)
+  val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone)
+  val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone)
+  val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone)
+  val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) }
+  val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock)
+
   val collect_inner_data = Reg(init=Bool(false))
   val (inner_data_cnt, inner_data_done) = 
     Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats)
   val (outer_data_cnt, outer_data_done) = 
     Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats)
 
-  val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients)))
+  val release_count = Reg(init=UInt(0, width = log2Up(nClients)))
   val probe_flags = Reg(init=Bits(0, width = nClients))
   val curr_p_id = PriorityEncoder(probe_flags)
 
   val pending_outer_write = co.messageHasData(xact)
   val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush)
-  val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)),
-                          { case TLId => outerId })
-  val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data),
-                          { case TLId => outerId })
-  val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)),
-                      { case TLId => outerId })
 
   val probe_initial_flags = Bits(width = nClients)
   probe_initial_flags := Bits(0)
@@ -260,26 +268,34 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
   val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0)))
   probe_initial_flags := ~(io.tile_incoherent | myflag)
 
-  io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data
+  io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && 
-  io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle)
+                              (state != s_idle) && 
+                              !collect_inner_data
+  io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && 
+                              (state != s_idle)
+
+  val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)),
+                          { case TLId => outerId })
+  val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data),
+                          { case TLId => outerId })
+  val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId })
 
   io.outer.acquire.valid := Bool(false)
+  io.outer.acquire.bits.payload := outer_read //default
   io.outer.acquire.bits.header.src := UInt(bankId)
-  //io.outer.acquire.bits.header.dst TODO
-  io.outer.acquire.bits.payload := outer_read 
   io.outer.grant.ready := io.inner.grant.ready
 
   io.inner.probe.valid := Bool(false)
   io.inner.probe.bits.header.src := UInt(bankId)
   io.inner.probe.bits.header.dst := curr_p_id
-  io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr)
+  io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr)
 
   io.inner.grant.valid := Bool(false)
   io.inner.grant.bits.header.src := UInt(bankId)
-  io.inner.grant.bits.header.dst := init_client_id
+  io.inner.grant.bits.header.dst := xact_src
-  io.inner.grant.bits.payload := Grant(xact.uncached,
+  io.inner.grant.bits.payload := Grant(xact_uncached,
                                         co.getGrantType(xact, co.masterMetadataOnFlush),
-                                        xact.client_xact_id,
+                                        xact_client_xact_id,
                                         UInt(trackerId),
                                         UInt(0)) // Data bypassed in parent
 
@@ -289,7 +305,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
   when(collect_inner_data) {
     io.inner.acquire.ready := Bool(true)
     when(io.inner.acquire.valid) {
-      data_ptrs(inner_data_cnt) := c_acq.payload.data
+      xact_data(inner_data_cnt) := c_acq.payload.data
     }
     when(inner_data_done) { collect_inner_data := Bool(false) }
   }
@@ -300,9 +316,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
       val needs_outer_write = co.messageHasData(c_acq.payload)
       val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush)
       when( io.inner.acquire.valid ) {
-        xact := c_acq.payload
+        xact_uncached := c_acq.payload.uncached
-        init_client_id := c_acq.header.src
+        xact_a_type := c_acq.payload.a_type
-        data_ptrs(UInt(0)) := c_acq.payload.data
+        xact_addr := c_acq.payload.addr
+        xact_client_xact_id := c_acq.payload.client_xact_id
+        xact_data(UInt(0)) := c_acq.payload.data
+        xact_subblock := c_acq.payload.subblock
+        xact_src := c_acq.header.src
         collect_inner_data := co.messageHasData(c_acq.payload)
         probe_flags := probe_initial_flags
         release_count := PopCount(probe_initial_flags)
@@ -319,12 +339,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
       }
 
       // Handle releases, which may have data to be written back
+      io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready
       when(io.inner.release.valid) {
         when(co.messageHasData(c_rel.payload)) {
           io.outer.acquire.valid := Bool(true)
           io.outer.acquire.bits.payload := outer_write_rel
           when(io.outer.acquire.ready) {
-            io.inner.release.ready := Bool(true)
             when(outer_data_done) {
               release_count := release_count - UInt(1)
               when(release_count === UInt(1)) {
@@ -334,7 +354,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
             }
           }
         } .otherwise {
-          io.inner.release.ready := Bool(true)
           release_count := release_count - UInt(1)
           when(release_count === UInt(1)) {
             state := Mux(pending_outer_write, s_mem_write, 
@@ -351,7 +370,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
       }
     }
     is(s_mem_write) {
-      io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
+      io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) 
+                                  else !collect_inner_data || (outer_data_cnt < inner_data_cnt))
       io.outer.acquire.bits.payload := outer_write_acq
       when(outer_data_done) {
         state := Mux(pending_outer_read, s_mem_read, s_make_grant)