reorganize moving non-submodule packages into src/main/scala

2016-08-19 10:58:56 -07:00
parent f78da0b0ea
commit 7b20609d4d
110 changed files with 3 additions and 381 deletions
--- a/src/main/scala/uncore/agents/Agents.scala
+++ b/src/main/scala/uncore/agents/Agents.scala
@ -0,0 +1,162 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import cde.{Parameters, Field}
+import junctions._
+import uncore.tilelink._
+import uncore.converters._
+import uncore.coherence._
+import uncore.util._
+
+case object NReleaseTransactors extends Field[Int]
+case object NProbeTransactors extends Field[Int]
+case object NAcquireTransactors extends Field[Int]
+
+trait HasCoherenceAgentParameters {
+  implicit val p: Parameters
+  val nReleaseTransactors = 1
+  val nAcquireTransactors = p(NAcquireTransactors)
+  val nTransactors = nReleaseTransactors + nAcquireTransactors
+  val blockAddrBits = p(PAddrBits) - p(CacheBlockOffsetBits)
+  val outerTLId = p(OuterTLId)
+  val outerTLParams = p(TLKey(outerTLId))
+  val outerDataBeats = outerTLParams.dataBeats
+  val outerDataBits = outerTLParams.dataBitsPerBeat
+  val outerBeatAddrBits = log2Up(outerDataBeats)
+  val outerByteAddrBits = log2Up(outerDataBits/8)
+  val outerWriteMaskBits = outerTLParams.writeMaskBits
+  val innerTLId = p(InnerTLId)
+  val innerTLParams = p(TLKey(innerTLId))
+  val innerDataBeats = innerTLParams.dataBeats
+  val innerDataBits = innerTLParams.dataBitsPerBeat
+  val innerWriteMaskBits = innerTLParams.writeMaskBits
+  val innerBeatAddrBits = log2Up(innerDataBeats)
+  val innerByteAddrBits = log2Up(innerDataBits/8)
+  val innerNCachingClients = innerTLParams.nCachingClients
+  val maxManagerXacts = innerTLParams.maxManagerXacts
+  require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement
+}
+
+abstract class CoherenceAgentModule(implicit val p: Parameters) extends Module
+  with HasCoherenceAgentParameters
+abstract class CoherenceAgentBundle(implicit val p: Parameters) extends junctions.ParameterizedBundle()(p)
+   with HasCoherenceAgentParameters
+
+trait HasCoherenceAgentWiringHelpers {
+  def doOutputArbitration[T <: TileLinkChannel](
+      out: DecoupledIO[T],
+      ins: Seq[DecoupledIO[T]]) {
+    def lock(o: T) = o.hasMultibeatData()
+    val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, Some(lock _)))
+    out <> arb.io.out
+    arb.io.in <> ins
+  }
+
+  def doInputRouting[T <: Bundle with HasManagerTransactionId](
+        in: DecoupledIO[T],
+        outs: Seq[DecoupledIO[T]]) {
+    val idx = in.bits.manager_xact_id
+    outs.map(_.bits := in.bits)
+    outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
+    in.ready := outs.map(_.ready).apply(idx)
+  }
+
+  /** Broadcasts valid messages on this channel to all trackers,
+    * but includes logic to allocate a new tracker in the case where
+    * no previously allocated tracker matches the new req's addr.
+    *
+    * When a match is reported, if ready is high the new transaction
+    * is merged; when ready is low the transaction is being blocked.
+    * When no match is reported, any high idles are presumed to be
+    * from trackers that are available for allocation, and one is
+    * assigned via alloc based on priority; if no idles are high then
+    * all trackers are busy with other transactions. If idle is high
+    * but ready is low, the tracker will be allocated but does not
+    * have sufficient buffering for the data.
+    */ 
+  def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData](
+        in: DecoupledIO[T],
+        outs: Seq[DecoupledIO[T]],
+        allocs: Seq[TrackerAllocation],
+        dataOverrides: Option[Seq[UInt]] = None,
+        allocOverride: Option[Bool] = None,
+        matchOverride: Option[Bool] = None) {
+    val ready_bits = outs.map(_.ready).asUInt
+    val can_alloc_bits = allocs.map(_.can).asUInt
+    val should_alloc_bits = PriorityEncoderOH(can_alloc_bits)
+    val match_bits = allocs.map(_.matches).asUInt
+    val no_matches = !match_bits.orR
+    val alloc_ok = allocOverride.getOrElse(Bool(true))
+    val match_ok = matchOverride.getOrElse(Bool(true))
+    in.ready := (Mux(no_matches, can_alloc_bits, match_bits) & ready_bits).orR && alloc_ok && match_ok
+    outs.zip(allocs).zipWithIndex.foreach { case((out, alloc), i) =>
+      out.valid := in.valid && match_ok && alloc_ok
+      out.bits := in.bits
+      dataOverrides foreach { d => out.bits.data := d(i) }
+      alloc.should := should_alloc_bits(i) && no_matches && alloc_ok
+    }
+  }
+}
+
+trait HasInnerTLIO extends HasCoherenceAgentParameters {
+  val inner = new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)}))
+  val incoherent = Vec(inner.tlNCachingClients, Bool()).asInput
+  def iacq(dummy: Int = 0) = inner.acquire.bits
+  def iprb(dummy: Int = 0) = inner.probe.bits
+  def irel(dummy: Int = 0) = inner.release.bits
+  def ignt(dummy: Int = 0) = inner.grant.bits
+  def ifin(dummy: Int = 0) = inner.finish.bits
+}
+
+trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters {
+  val outer = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))
+  def oacq(dummy: Int = 0) = outer.acquire.bits
+  def ognt(dummy: Int = 0) = outer.grant.bits
+}
+
+trait HasCachedOuterTLIO extends HasCoherenceAgentParameters {
+  val outer = new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))
+  def oacq(dummy: Int = 0) = outer.acquire.bits
+  def oprb(dummy: Int = 0) = outer.probe.bits
+  def orel(dummy: Int = 0) = outer.release.bits
+  def ognt(dummy: Int = 0) = outer.grant.bits
+}
+
+class ManagerTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+  with HasInnerTLIO
+  with HasUncachedOuterTLIO
+
+abstract class CoherenceAgent(implicit p: Parameters) extends CoherenceAgentModule()(p) {
+  def innerTL: ManagerTileLinkIO
+  def outerTL: ClientTileLinkIO
+  def incoherent: Vec[Bool]
+}
+
+abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p)
+    with HasCoherenceAgentWiringHelpers {
+  val io = new ManagerTLIO
+  def innerTL = io.inner
+  def outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)}))
+  def incoherent = io.incoherent
+}
+
+class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+  with HasInnerTLIO
+  with HasCachedOuterTLIO
+
+abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p)
+    with HasCoherenceAgentWiringHelpers {
+  val io = new HierarchicalTLIO
+  def innerTL = io.inner
+  def outerTL = io.outer
+  def incoherent = io.incoherent
+
+  // TODO: Remove this function (and all its calls) when we support probing the L2
+  def disconnectOuterProbeAndFinish() {
+    io.outer.probe.ready := Bool(false)
+    io.outer.finish.valid := Bool(false)
+    assert(!io.outer.probe.valid, "L2 agent got illegal probe")
+  }
+}
--- a/src/main/scala/uncore/agents/Broadcast.scala
+++ b/src/main/scala/uncore/agents/Broadcast.scala
@ -0,0 +1,204 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.constants._
+import uncore.util._
+import cde.Parameters
+
+class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
+
+  // Create TSHRs for outstanding transactions
+  val irelTrackerList =
+    (0 until nReleaseTransactors).map(id =>
+      Module(new BufferedBroadcastVoluntaryReleaseTracker(id)))
+  val iacqTrackerList = 
+    (nReleaseTransactors until nTransactors).map(id =>
+      Module(new BufferedBroadcastAcquireTracker(id)))
+  val trackerList = irelTrackerList ++ iacqTrackerList
+
+  // Propagate incoherence flags
+  trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
+
+  // Create an arbiter for the one memory port
+  val outerList = trackerList.map(_.io.outer)
+  val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
+                                                    (p.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> outerList
+  io.outer <> outer_arb.io.out
+
+  // Handle acquire transaction initiation
+  val irel_vs_iacq_conflict =
+    io.inner.acquire.valid &&
+    io.inner.release.valid &&
+    io.irel().conflicts(io.iacq())
+
+  doInputRoutingWithAllocation(
+    in = io.inner.acquire,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc.iacq),
+    allocOverride = Some(!irel_vs_iacq_conflict))
+
+  // Handle releases, which might be voluntary and might have data
+  doInputRoutingWithAllocation(
+    in = io.inner.release,
+    outs = trackerList.map(_.io.inner.release),
+    allocs = trackerList.map(_.io.alloc.irel))
+
+  // Wire probe requests and grant reply to clients, finish acks from clients
+  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
+
+  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
+
+  doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
+
+  disconnectOuterProbeAndFinish()
+}
+
+class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+}
+
+trait BroadcastsToAllClients extends HasCoherenceAgentParameters {
+  val coh = HierarchicalMetadata.onReset
+  val inner_coh = coh.inner
+  val outer_coh = coh.outer
+  def full_representation = ~UInt(0, width = innerNCachingClients)
+}
+
+abstract class BroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends VoluntaryReleaseTracker(trackerId)(p) 
+    with EmitsVoluntaryReleases
+    with BroadcastsToAllClients {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+
+  // Checks for illegal behavior
+  assert(!(state === s_idle && io.inner.release.fire() && io.alloc.irel.should && !io.irel().isVoluntary()),
+    "VoluntaryReleaseTracker accepted Release that wasn't voluntary!")
+}
+
+abstract class BroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends AcquireTracker(trackerId)(p) 
+    with EmitsVoluntaryReleases
+    with BroadcastsToAllClients {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+
+  val alwaysWriteFullBeat = false
+  val nSecondaryMisses = 1
+  def iacq_can_merge = Bool(false)
+
+  // Checks for illegal behavior
+  // TODO: this could be allowed, but is a useful check against allocation gone wild
+  assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq.should &&
+    io.iacq().hasMultibeatData() && !io.iacq().first()),
+    "AcquireTracker initialized with a tail data beat.")
+
+  assert(!(state =/= s_idle && pending_ignt && xact_iacq.isPrefetch()),
+    "Broadcast Hub does not support Prefetches.")
+
+  assert(!(state =/= s_idle && pending_ignt && xact_iacq.isAtomic()),
+    "Broadcast Hub does not support PutAtomics.")
+}
+
+class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastVoluntaryReleaseTracker(trackerId)(p)
+    with HasDataBuffer {
+
+  // Tell the parent if any incoming messages conflict with the ongoing transaction
+  routeInParent(irelCanAlloc = Bool(true))
+
+  // Start transaction by accepting inner release
+  innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
+
+  // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
+  io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact
+
+  when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data }
+
+  // Dispatch outer release
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    data = data_buffer(vol_ognt_counter.up.idx),
+    add_pending_send_bit = irel_is_allocating)
+
+  quiesce() {}
+}
+
+class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastAcquireTracker(trackerId)(p)
+    with HasByteWriteMaskBuffer {
+
+  // Setup IOs used for routing in the parent
+  routeInParent(iacqCanAlloc = Bool(true))
+
+  // First, take care of accpeting new acquires or secondary misses
+  // Handling of primary and secondary misses' data and write mask merging
+  innerAcquire(
+    can_alloc = Bool(false),
+    next = s_inner_probe)
+
+  io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact_multibeat
+
+  // Track which clients yet need to be probed and make Probe message
+  // If a writeback occurs, we can forward its data via the buffer,
+  // and skip having to go outwards
+  val skip_outer_acquire = pending_ignt_data.andR
+
+  innerProbe(
+    inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
+    Mux(!skip_outer_acquire, s_outer_acquire, s_busy))
+
+  // Handle incoming releases from clients, which may reduce sharer counts
+  // and/or write back dirty data, and may be unexpected voluntary releases
+  def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
+                         io.irel().isVoluntary() &&
+                         !state.isOneOf(s_idle, s_meta_write) &&
+                         !all_pending_done &&
+                         !io.outer.grant.fire() &&
+                         !io.inner.grant.fire() &&
+                         !vol_ignt_counter.pending &&
+                         !blockInnerRelease()
+
+  innerRelease(block_vol_ignt = vol_ognt_counter.pending) 
+
+  //TODO: accept vol irels when state === s_idle, operate like the VolRelTracker
+  io.inner.release.ready := irel_can_merge || irel_same_xact
+
+  mergeDataInner(io.inner.release)
+
+  // If there was a writeback, forward it outwards
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    data = data_buffer(vol_ognt_counter.up.idx))
+
+  // Send outer request for miss
+  outerAcquire(
+    caching = !xact_iacq.isBuiltInType(),
+    coh = outer_coh,
+    data = data_buffer(ognt_counter.up.idx),
+    wmask = wmask_buffer(ognt_counter.up.idx),
+    next = s_busy)
+    
+  // Handle the response from outer memory
+  mergeDataOuter(io.outer.grant)
+
+  // Acknowledge or respond with data
+  innerGrant(
+    data = data_buffer(ignt_data_idx),
+    external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending)
+
+  when(iacq_is_allocating) {
+    initializeProbes()
+  }
+
+  initDataInner(io.inner.acquire, iacq_is_allocating || iacq_is_merging)
+
+  // Wait for everything to quiesce
+  quiesce() { clearWmaskBuffer() }
+}
--- a/src/main/scala/uncore/agents/Bufferless.scala
+++ b/src/main/scala/uncore/agents/Bufferless.scala
@ -0,0 +1,162 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.constants._
+import cde.Parameters
+
+
+class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
+
+  // Create TSHRs for outstanding transactions
+  val irelTrackerList =
+    (0 until nReleaseTransactors).map(id =>
+      Module(new BufferlessBroadcastVoluntaryReleaseTracker(id)))
+  val iacqTrackerList = 
+    (nReleaseTransactors until nTransactors).map(id =>
+      Module(new BufferlessBroadcastAcquireTracker(id)))
+  val trackerList = irelTrackerList ++ iacqTrackerList
+
+  // Propagate incoherence flags
+  trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
+
+  // Create an arbiter for the one memory port
+  val outerList = trackerList.map(_.io.outer)
+  val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
+                                                    (p.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> outerList
+  io.outer <> outer_arb.io.out
+
+  val iacq = Queue(io.inner.acquire, 1, pipe=true)
+  val irel = Queue(io.inner.release, 1, pipe=true)
+
+  // Handle acquire transaction initiation
+  val irel_vs_iacq_conflict =
+    iacq.valid &&
+    irel.valid &&
+    irel.bits.conflicts(iacq.bits)
+
+  doInputRoutingWithAllocation(
+    in = iacq,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc.iacq),
+    allocOverride = Some(!irel_vs_iacq_conflict))
+  io.outer.acquire.bits.data := iacq.bits.data
+  when (io.oacq().hasData()) {
+    io.outer.acquire.bits.addr_beat := iacq.bits.addr_beat
+  }
+
+  // Handle releases, which might be voluntary and might have data
+  doInputRoutingWithAllocation(
+    in = irel,
+    outs = trackerList.map(_.io.inner.release),
+    allocs = trackerList.map(_.io.alloc.irel))
+  io.outer.release.bits.data := irel.bits.data
+  when (io.orel().hasData()) {
+    io.outer.release.bits.addr_beat := irel.bits.addr_beat
+  }
+
+  // Wire probe requests and grant reply to clients, finish acks from clients
+  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
+
+  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
+  io.inner.grant.bits.data := io.outer.grant.bits.data
+  io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat
+
+  doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
+
+  disconnectOuterProbeAndFinish()
+}
+
+class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastVoluntaryReleaseTracker(trackerId)(p) {
+
+  // Tell the parent if any incoming messages conflict with the ongoing transaction
+  routeInParent(irelCanAlloc = Bool(true))
+
+  // Start transaction by accepting inner release
+  innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
+
+  // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
+  // and if the outer relase path is clear 
+  io.inner.release.ready := Mux(io.irel().hasData(),
+    (state =/= s_idle) && (irel_can_merge || irel_same_xact) && io.outer.release.ready,
+    (state === s_idle) || irel_can_merge || irel_same_xact)
+
+  // Dispatch outer release
+  outerRelease(coh = outer_coh.onHit(M_XWR), buffering = Bool(false))
+
+  quiesce() {}
+}
+
+class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastAcquireTracker(trackerId)(p) {
+
+  // Setup IOs used for routing in the parent
+  routeInParent(iacqCanAlloc = Bool(true))
+
+  // First, take care of accpeting new acquires or secondary misses
+  // Handling of primary and secondary misses' data and write mask merging
+  innerAcquire(
+    can_alloc = Bool(false),
+    next = s_inner_probe)
+
+  // We are never going to merge anything in the bufferless hub
+  // Therefore, we only need to concern ourselves with the allocated
+  // transaction and (in case of PutBlock) subsequent tail beats
+  val iacq_can_forward = iacq_same_xact && !vol_ognt_counter.pending
+  io.inner.acquire.ready := Mux(io.iacq().hasData(),
+    state === s_outer_acquire && iacq_can_forward && io.outer.acquire.ready,
+    state === s_idle && io.alloc.iacq.should)
+
+  // Track which clients yet need to be probed and make Probe message
+  innerProbe(
+    inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
+    s_outer_acquire)
+
+  // Handle incoming releases from clients, which may reduce sharer counts
+  // and/or write back dirty data, and may be unexpected voluntary releases
+  def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
+                         io.irel().isVoluntary() &&
+                         !vol_ignt_counter.pending &&
+                         !(io.irel().hasData() && ognt_counter.pending) &&
+                         (state =/= s_idle)
+
+  innerRelease(block_vol_ignt = vol_ognt_counter.pending) 
+
+  val irel_could_accept = irel_can_merge || irel_same_xact
+  io.inner.release.ready := irel_could_accept &&
+    (!io.irel().hasData() || io.outer.release.ready)
+
+  // If there was a writeback, forward it outwards
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    buffering = Bool(false),
+    block_orel = !irel_could_accept)
+
+  // Send outer request for miss
+  outerAcquire(
+    caching = !xact_iacq.isBuiltInType(),
+    block_outer_acquire = vol_ognt_counter.pending,
+    buffering = Bool(false),
+    coh = outer_coh,
+    next = s_busy)
+
+  // Handle the response from outer memory
+  when (ognt_counter.pending && io.ognt().hasData()) {
+    io.outer.grant.ready := io.inner.grant.ready // bypass data
+  }
+
+  // Acknowledge or respond with data
+  innerGrant(
+    external_pending = pending_orel || vol_ognt_counter.pending,
+    buffering = Bool(false))
+
+  when(iacq_is_allocating) { initializeProbes() }
+
+  // Wait for everything to quiesce
+  quiesce() {}
+}
--- a/src/main/scala/uncore/agents/Cache.scala
+++ b/src/main/scala/uncore/agents/Cache.scala
--- a/src/main/scala/uncore/agents/Ecc.scala
+++ b/src/main/scala/uncore/agents/Ecc.scala
@ -0,0 +1,146 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.util._
+
+abstract class Decoding
+{
+  def uncorrected: UInt
+  def corrected: UInt
+  def correctable: Bool
+  def uncorrectable: Bool
+  def error = correctable || uncorrectable
+}
+
+abstract class Code
+{
+  def width(w0: Int): Int
+  def encode(x: UInt): UInt
+  def decode(x: UInt): Decoding
+}
+
+class IdentityCode extends Code
+{
+  def width(w0: Int) = w0
+  def encode(x: UInt) = x
+  def decode(y: UInt) = new Decoding {
+    def uncorrected = y
+    def corrected = y
+    def correctable = Bool(false)
+    def uncorrectable = Bool(false)
+  }
+}
+
+class ParityCode extends Code
+{
+  def width(w0: Int) = w0+1
+  def encode(x: UInt) = Cat(x.xorR, x)
+  def decode(y: UInt) = new Decoding {
+    def uncorrected = y(y.getWidth-2,0)
+    def corrected = uncorrected
+    def correctable = Bool(false)
+    def uncorrectable = y.xorR
+  }
+}
+
+class SECCode extends Code
+{
+  def width(k: Int) = {
+    val m = log2Floor(k) + 1
+    k + m + (if((1 << m) < m+k+1) 1 else 0)
+  }
+  def encode(x: UInt) = {
+    val k = x.getWidth
+    require(k > 0)
+    val n = width(k)
+
+    val y = for (i <- 1 to n) yield {
+      if (isPow2(i)) {
+        val r = for (j <- 1 to n; if j != i && (j & i) != 0)
+          yield x(mapping(j))
+        r reduce (_^_)
+      } else
+        x(mapping(i))
+    }
+    y.asUInt
+  }
+  def decode(y: UInt) = new Decoding {
+    val n = y.getWidth
+    require(n > 0 && !isPow2(n))
+
+    val p2 = for (i <- 0 until log2Up(n)) yield 1 << i
+    val syndrome = (p2 map { i =>
+      val r = for (j <- 1 to n; if (j & i) != 0)
+        yield y(j-1)
+      r reduce (_^_)
+    }).asUInt
+
+    private def swizzle(z: UInt) = (1 to n).filter(i => !isPow2(i)).map(i => z(i-1)).asUInt
+    def uncorrected = swizzle(y)
+    def corrected = swizzle(((y << 1) ^ UIntToOH(syndrome)) >> 1)
+    def correctable = syndrome.orR
+    def uncorrectable = Bool(false)
+  }
+  private def mapping(i: Int) = i-1-log2Up(i)
+}
+
+class SECDEDCode extends Code
+{
+  private val sec = new SECCode
+  private val par = new ParityCode
+
+  def width(k: Int) = sec.width(k)+1
+  def encode(x: UInt) = par.encode(sec.encode(x))
+  def decode(x: UInt) = new Decoding {
+    val secdec = sec.decode(x(x.getWidth-2,0))
+    val pardec = par.decode(x)
+
+    def uncorrected = secdec.uncorrected
+    def corrected = secdec.corrected
+    def correctable = pardec.uncorrectable
+    def uncorrectable = !pardec.uncorrectable && secdec.correctable
+  }
+}
+
+object ErrGen
+{
+  // generate a 1-bit error with approximate probability 2^-f
+  def apply(width: Int, f: Int): UInt = {
+    require(width > 0 && f >= 0 && log2Up(width) + f <= 16)
+    UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0)
+  }
+  def apply(x: UInt, f: Int): UInt = x ^ apply(x.getWidth, f)
+}
+
+class SECDEDTest extends Module
+{
+  val code = new SECDEDCode
+  val k = 4
+  val n = code.width(k)
+
+  val io = new Bundle {
+    val original = Bits(OUTPUT, k)
+    val encoded = Bits(OUTPUT, n)
+    val injected = Bits(OUTPUT, n)
+    val uncorrected = Bits(OUTPUT, k)
+    val corrected = Bits(OUTPUT, k)
+    val correctable = Bool(OUTPUT)
+    val uncorrectable = Bool(OUTPUT)
+  }
+
+  val c = Counter(Bool(true), 1 << k)
+  val numErrors = Counter(c._2, 3)._1
+  val e = code.encode(c._1)
+  val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1))
+  val d = code.decode(i)
+
+  io.original := c._1
+  io.encoded := e
+  io.injected := i
+  io.uncorrected := d.uncorrected
+  io.corrected := d.corrected
+  io.correctable := d.correctable
+  io.uncorrectable := d.uncorrectable
+}
--- a/src/main/scala/uncore/agents/Mmio.scala
+++ b/src/main/scala/uncore/agents/Mmio.scala
@ -0,0 +1,73 @@
+package uncore.agents
+
+import Chisel._
+import uncore.tilelink._
+import cde.Parameters
+
+class MMIOTileLinkManagerData(implicit p: Parameters)
+    extends TLBundle()(p)
+    with HasClientId
+    with HasClientTransactionId
+
+class MMIOTileLinkManager(implicit p: Parameters)
+    extends CoherenceAgentModule()(p) {
+  val io = new ManagerTLIO
+
+  // MMIO requests should never need probe or release
+  io.inner.probe.valid := Bool(false)
+  io.inner.release.ready := Bool(false)
+
+  val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData()
+  val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0)
+  val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1)
+
+  // Acquire and Grant are basically passthru,
+  // except client_id and client_xact_id need to be converted.
+  // Associate the inner client_id and client_xact_id
+  // with the outer client_xact_id.
+  val xact_pending = Reg(init = UInt(0, maxManagerXacts))
+  val xact_id_sel = PriorityEncoder(~xact_pending)
+  val xact_id_reg = RegEnable(xact_id_sel, multibeat_start)
+  val xact_multibeat = Reg(init = Bool(false))
+  val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel)
+  val xact_free = !xact_pending.andR
+  val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData))
+
+  io.inner.acquire.ready := io.outer.acquire.ready && xact_free
+  io.outer.acquire.valid := io.inner.acquire.valid && xact_free
+  io.outer.acquire.bits  := io.inner.acquire.bits
+  io.outer.acquire.bits.client_xact_id := outer_xact_id
+
+  def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool =
+    !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1)
+
+  def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt =
+    Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0))
+
+  def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt =
+    ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(),
+      UIntToOH(in.bits.manager_xact_id), UInt(0))
+
+  def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt =
+    ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0))
+
+  xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) &
+                                  clearPendingBitOnFin(io.inner.finish) &
+                                  clearPendingBitOnGnt(io.inner.grant)
+
+  when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) {
+    xact_buffer(outer_xact_id) := io.iacq()
+  }
+
+  when (multibeat_start) { xact_multibeat := Bool(true) }
+  when (multibeat_end)   { xact_multibeat := Bool(false) }
+
+  val gnt_xact = xact_buffer(io.ognt().client_xact_id)
+  io.outer.grant.ready := io.inner.grant.ready
+  io.inner.grant.valid := io.outer.grant.valid
+  io.inner.grant.bits  := io.outer.grant.bits
+  io.inner.grant.bits.client_id := gnt_xact.client_id
+  io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id
+  io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id
+  io.inner.finish.ready := Bool(true)
+}
--- a/src/main/scala/uncore/agents/StatelessBridge.scala
+++ b/src/main/scala/uncore/agents/StatelessBridge.scala
@ -0,0 +1,69 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.constants._
+import uncore.devices._
+import cde.{Parameters, Field, Config}
+
+/** The ManagerToClientStateless Bridge does not maintain any state for the messages
+  *  which pass through it. It simply passes the messages back and forth without any
+  *  tracking or translation. 
+  *  
+  * This can reduce area and timing in very constrained situations:
+  *   - The Manager and Client implement the same coherence protocol
+  *   - There are no probe or finish messages.
+  *   - The outer transaction ID is large enough to handle all possible inner
+  *     transaction IDs, such that no remapping state must be maintained.
+  *
+  * This bridge DOES NOT keep the uncached channel coherent with the cached
+  * channel. Uncached requests to blocks cached by the L1 will not probe the L1.
+  * As a result, uncached reads to cached blocks will get stale data until
+  * the L1 performs a voluntary writeback, and uncached writes to cached blocks
+  * will get lost, as the voluntary writeback from the L1 will overwrite the
+  * changes. If your tile relies on probing the L1 data cache in order to
+  * share data between the instruction cache and data cache (e.g. you are using
+  * a non-blocking L1 D$) or if the tile has uncached channels capable of
+  * writes (e.g. Hwacha and other RoCC accelerators), DO NOT USE THIS BRIDGE.
+  */
+
+class ManagerToClientStatelessBridge(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
+  val icid = io.inner.tlClientIdBits
+  val ixid = io.inner.tlClientXactIdBits
+  val oxid = io.outer.tlClientXactIdBits
+
+  val innerCoh = io.inner.tlCoh.getClass
+  val outerCoh = io.outer.tlCoh.getClass
+
+  // Stateless Bridge is only usable in certain constrained situations.
+  // Sanity check its usage here.
+
+  require(io.inner.tlNCachingClients <= 1)
+  require(icid + ixid <= oxid)
+  require(innerCoh eq outerCoh,
+    s"Coherence policies do not match: inner is ${innerCoh.getSimpleName}, outer is ${outerCoh.getSimpleName}")
+
+  io.outer.acquire.valid := io.inner.acquire.valid
+  io.inner.acquire.ready := io.outer.acquire.ready
+  io.outer.acquire.bits := io.inner.acquire.bits
+  io.outer.acquire.bits.client_xact_id := Cat(io.inner.acquire.bits.client_id, io.inner.acquire.bits.client_xact_id)
+
+  io.outer.release.valid := io.inner.release.valid
+  io.inner.release.ready := io.outer.release.ready
+  io.outer.release.bits := io.inner.release.bits
+  io.outer.release.bits.client_xact_id := Cat(io.inner.release.bits.client_id, io.inner.release.bits.client_xact_id)
+
+  io.inner.grant.valid := io.outer.grant.valid
+  io.outer.grant.ready := io.inner.grant.ready
+  io.inner.grant.bits := io.outer.grant.bits
+  io.inner.grant.bits.client_xact_id := io.outer.grant.bits.client_xact_id(ixid-1, 0)
+  io.inner.grant.bits.client_id := io.outer.grant.bits.client_xact_id(icid+ixid-1, ixid)
+
+  io.inner.probe.valid := Bool(false)
+  io.inner.finish.ready := Bool(true)
+
+  disconnectOuterProbeAndFinish()
+}
--- a/src/main/scala/uncore/agents/StoreDataQueue.scala
+++ b/src/main/scala/uncore/agents/StoreDataQueue.scala
@ -0,0 +1,119 @@
+// See LICENSE for license details.
+
+package uncore.agents
+import Chisel._
+import uncore.tilelink._
+import cde.{Parameters, Field}
+
+case object L2StoreDataQueueDepth extends Field[Int]
+
+trait HasStoreDataQueueParameters extends HasCoherenceAgentParameters {
+  val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats
+  val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
+  val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
+}
+
+class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+    with HasStoreDataQueueParameters {
+  val idx = UInt(width = dqIdxBits)
+  val loc = UInt(width = log2Ceil(nDataQueueLocations))
+} 
+
+object DataQueueLocation {
+  def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = {
+    val d = Wire(new DataQueueLocation)
+    d.idx := idx
+    d.loc := loc
+    d
+  }
+}
+
+trait HasStoreDataQueue extends HasStoreDataQueueParameters {
+  val io: HierarchicalTLIO
+  val trackerIOsList: Seq[HierarchicalXactTrackerIO]
+
+  val internalDataBits = new DataQueueLocation().getWidth
+  val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
+
+  val usingStoreDataQueue = p.alterPartial({
+    case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
+    case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
+  })
+
+  // Queue to store impending Put data
+  lazy val sdq = Reg(Vec(sdqDepth, io.iacq().data))
+  lazy val sdq_val = Reg(init=Bits(0, sdqDepth))
+  lazy val sdq_alloc_id = PriorityEncoder(~sdq_val)
+  lazy val sdq_rdy = !sdq_val.andR
+  lazy val sdq_enq = trackerIOsList.map( t =>
+                  (t.alloc.iacq.should || t.alloc.iacq.matches) &&
+                    t.inner.acquire.fire() &&
+                    t.iacq().hasData()
+                ).reduce(_||_)
+
+  lazy val sdqLoc = List.fill(nTransactors) {
+    DataQueueLocation(sdq_alloc_id, inStoreQueue).asUInt
+  }
+
+  /*
+  doInputRoutingWithAllocation(
+    in = io.inner.acquire,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc._iacq),
+    dataOverride = Some(sdqLoc),
+    allocOverride = Some(sdq_rdy && !irel_vs_iacq_conflict))
+  */
+
+  // Queue to store impending Voluntary Release data
+  lazy val voluntary = io.irel().isVoluntary()
+  lazy val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData()
+  lazy val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
+  lazy val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1
+  
+
+  lazy val vwbqLoc = (0 until nTransactors).map(i =>
+    (DataQueueLocation(rel_data_cnt,
+                       (if(i < nReleaseTransactors) inVolWBQueue
+                        else inClientReleaseQueue)).asUInt))
+  /*
+  doInputRoutingWithAllocation(
+    io.inner.release,
+    trackerList.map(_.io.inner.release),
+    trackerList.map(_.io.matches.irel),
+    trackerList.map(_.io.alloc.irel),
+    Some(vwbqLoc))
+  */
+
+  val outer_arb: ClientTileLinkIOArbiter
+  lazy val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data)
+  /*
+  val outer_arb = Module(new ClientTileLinkIOArbiter(trackerList.size)
+                    (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> trackerList
+  */
+  // Get the pending data out of the store data queue
+  lazy val is_in_sdq = outer_data_ptr.loc === inStoreQueue
+  lazy val free_sdq = io.outer.acquire.fire() &&
+                  io.outer.acquire.bits.hasData() &&
+                  outer_data_ptr.loc === inStoreQueue
+  /*
+  io.outer <> outer_arb.io.out
+  io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array(
+                                          inStoreQueue -> sdq(outer_data_ptr.idx),
+                                          inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
+  */
+
+  // Enqueue SDQ data
+  def sdqEnqueue() {
+    when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data }
+    when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data }
+  }
+
+  // Update SDQ valid bits
+  def sdqUpdate() {
+    when (io.outer.acquire.valid || sdq_enq) {
+      sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | 
+                 PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
+    }
+  }
+}
--- a/src/main/scala/uncore/agents/Trackers.scala
+++ b/src/main/scala/uncore/agents/Trackers.scala
@ -0,0 +1,654 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.util._
+import uncore.util._
+import junctions._
+import cde.{Field, Parameters}
+import scala.math.max
+
+case object EnableL2Logging extends Field[Boolean]
+
+class TrackerAllocation extends Bundle {
+  val matches = Bool(OUTPUT)
+  val can = Bool(OUTPUT)
+  val should = Bool(INPUT)
+}
+
+class TrackerAllocationIO(implicit val p: Parameters)
+    extends ParameterizedBundle()(p)
+    with HasCacheBlockAddress {
+  val iacq = new TrackerAllocation
+  val irel = new TrackerAllocation
+  val oprb = new TrackerAllocation
+  val idle = Bool(OUTPUT)
+  override val addr_block = UInt(OUTPUT, tlBlockAddrBits)
+}
+
+trait HasTrackerAllocationIO extends Bundle {
+  implicit val p: Parameters
+  val alloc = new TrackerAllocationIO
+}
+
+class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p)
+  with HasTrackerAllocationIO
+
+class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p)
+  with HasTrackerAllocationIO
+
+abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p)
+    with HasXactTrackerStates
+    with HasPendingBitHelpers {
+  override val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9)
+  val state = Reg(init=s_idle)
+
+  def quiesce(next: UInt = s_idle)(restore: => Unit) {
+    all_pending_done := !scoreboard.foldLeft(Bool(false))(_||_)
+    when(state === s_busy && all_pending_done) {
+      state := next
+      restore
+    }
+  }
+
+  def pinAllReadyValidLow[T <: Data](b: Bundle) {
+    b.elements.foreach {
+      _._2 match {
+        case d: DecoupledIO[_] =>
+          if(d.ready.dir == OUTPUT) d.ready := Bool(false)
+          else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
+        case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) 
+        case b: Bundle => pinAllReadyValidLow(b)
+        case _ =>
+      }
+    }
+  }
+}
+
+trait HasXactTrackerStates {
+  def state: UInt 
+  def s_idle: UInt = UInt(0)
+  def s_meta_read: UInt
+  def s_meta_resp: UInt
+  def s_wb_req: UInt
+  def s_wb_resp: UInt
+  def s_inner_probe: UInt
+  def s_outer_acquire: UInt
+  def s_busy: UInt
+  def s_meta_write: UInt
+}
+
+trait HasPendingBitHelpers extends HasDataBeatCounters {
+  val scoreboard = scala.collection.mutable.ListBuffer.empty[Bool]
+  val all_pending_done = Wire(Bool())
+
+  def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt =
+    Fill(in.tlDataBeats, inc) &  UIntToOH(in.addr_beat)
+
+  def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt =
+    ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat)
+
+  def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt =
+    Fill(in.tlNCachingClients, inc) &  UIntToOH(in.client_id)
+
+  def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt =
+    ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id)
+
+  def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt =
+    addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits)
+
+  def addPendingBitWhenBeatHasDataAndAllocs(
+      in: DecoupledIO[AcquireFromSrc],
+      alloc_override: Bool = Bool(false)): UInt =
+    addPendingBitWhenBeatHasData(in, in.bits.allocate() || alloc_override)
+
+  def addPendingBitWhenBeatNeedsRead(in: DecoupledIO[AcquireFromSrc],
+      always: Bool = Bool(true), unless: Bool = Bool(false)): UInt = {
+    val a = in.bits
+    val needs_read = !unless && (a.isGet() || a.isAtomic() || a.hasPartialWritemask()) || always
+    addPendingBitWhenBeat(in.fire() && needs_read, a)
+  }
+
+  def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt =
+    addPendingBitWhenBeat(in.fire() && in.bits.hasPartialWritemask(), in.bits)
+
+  def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt =
+    Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat))
+
+  def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits)
+
+  def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenId(in.fire(), in.bits)
+
+  def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
+
+  def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt =
+    addPendingBitWhenId(in.fire(), in.bits)
+
+  def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
+    addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
+
+  def addOtherBits(en: Bool, nBits: Int): UInt =
+    Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits))
+
+  def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
+    addOtherBits(in.fire() &&
+                 in.bits.hasMultibeatData() &&
+                 in.bits.addr_beat === UInt(0),
+                 in.bits.tlDataBeats)
+
+  def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
+    ~addPendingBitsOnFirstBeat(in)
+}
+
+trait HasDataBuffer extends HasCoherenceAgentParameters {
+  val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits)))
+
+  type TLDataBundle = TLBundle with HasTileLinkData with HasTileLinkBeatId
+
+  def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) {
+    when(in.fire() && in.bits.hasData() && alloc) { 
+      data_buffer(in.bits.addr_beat) := in.bits.data
+    }
+  }
+
+  // TODO: provide func for accessing when innerDataBeats =/= outerDataBeats or internalDataBeats
+  def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
+    data_buffer(beat) := incoming 
+  }
+
+  def mergeDataInner[T <: TLDataBundle](in: DecoupledIO[T]) {
+    when(in.fire() && in.bits.hasData()) { 
+      mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data)
+    }
+  }
+
+  def mergeDataOuter[T <: TLDataBundle](in: DecoupledIO[T]) {
+    when(in.fire() && in.bits.hasData()) { 
+      mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data)
+    }
+  }
+}
+
+trait HasByteWriteMaskBuffer extends HasDataBuffer { 
+  val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerWriteMaskBits)))
+  val data_valid = Vec(wmask_buffer.map(wmask => wmask.andR))
+
+  override def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) {
+    when(in.fire() && in.bits.hasData() && alloc) { 
+      val beat = in.bits.addr_beat
+      val full = FillInterleaved(8, in.bits.wmask())
+      data_buffer(beat) := (~full & data_buffer(beat)) | (full & in.bits.data)
+      wmask_buffer(beat) := in.bits.wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed
+    }
+  }
+
+  override def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
+    val old_data = incoming     // Refilled, written back, or de-cached data
+    val new_data = data_buffer(beat) // Newly Put data is already in the buffer
+    val wmask = FillInterleaved(8, wmask_buffer(beat))
+    data_buffer(beat) := (~wmask & old_data) | (wmask & new_data)
+    wmask_buffer(beat) := ~UInt(0, innerWriteMaskBits)
+  }
+
+  def clearWmaskBuffer() {
+    wmask_buffer.foreach { w => w := UInt(0) }
+  }
+}
+
+trait HasBlockAddressBuffer extends HasCoherenceAgentParameters {
+  val xact_addr_block = Reg(init = UInt(0, width = blockAddrBits))
+}
+
+
+trait HasAcquireMetadataBuffer extends HasBlockAddressBuffer {
+  val xact_allocate = Reg{ Bool() }
+  val xact_amo_shift_bytes = Reg{ UInt() }
+  val xact_op_code = Reg{ UInt() }
+  val xact_addr_byte = Reg{ UInt() }
+  val xact_op_size = Reg{ UInt() }
+  val xact_addr_beat = Wire(UInt())
+  val xact_iacq = Wire(new SecondaryMissInfo)
+}
+
+trait HasVoluntaryReleaseMetadataBuffer extends HasBlockAddressBuffer
+    with HasPendingBitHelpers
+    with HasXactTrackerStates {
+  def io: HierarchicalXactTrackerIO
+
+  val xact_vol_ir_r_type = Reg{ UInt() }
+  val xact_vol_ir_src = Reg{ UInt() }
+  val xact_vol_ir_client_xact_id = Reg{ UInt() }
+
+  def xact_vol_irel = Release(
+                        src = xact_vol_ir_src,
+                        voluntary = Bool(true),
+                        r_type = xact_vol_ir_r_type,
+                        client_xact_id = xact_vol_ir_client_xact_id,
+                        addr_block = xact_addr_block)
+                        (p.alterPartial({ case TLId => p(InnerTLId) }))
+}
+
+trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
+  def inner_coh: ManagerMetadata
+
+  val pending_irel_data = Reg(init=Bits(0, width = innerDataBeats))
+  val vol_ignt_counter = Wire(new TwoWayBeatCounterStatus)
+
+  def irel_can_merge: Bool
+  def irel_same_xact: Bool
+  def irel_is_allocating: Bool = state === s_idle && io.alloc.irel.should && io.inner.release.valid
+  def irel_is_merging: Bool = (irel_can_merge || irel_same_xact) && io.inner.release.valid
+
+  def innerRelease(block_vol_ignt: Bool = Bool(false), next: UInt = s_busy) {
+    connectTwoWayBeatCounters(
+      status = vol_ignt_counter,
+      up = io.inner.release,
+      down = io.inner.grant,
+      trackUp = (r: Release) => {
+        Mux(state === s_idle, io.alloc.irel.should, io.alloc.irel.matches) && r.isVoluntary() && r.requiresAck()
+      },
+      trackDown = (g: Grant) => (state =/= s_idle) && g.isVoluntary())
+
+
+    when(irel_is_allocating) {
+      xact_addr_block := io.irel().addr_block
+      // Set all of them to pending in the beginning as a precaution
+      // If it turns out we don't need some or all of the beats, they will
+      // be overridden below
+      pending_irel_data := ~UInt(0, innerDataBeats)
+      state := next
+    }
+
+    val irel_fire = (irel_is_allocating || irel_is_merging) && io.inner.release.ready
+    when (irel_fire) {
+      when (io.irel().first()) {
+        when (io.irel().isVoluntary()) {
+          xact_vol_ir_r_type := io.irel().r_type
+          xact_vol_ir_src := io.irel().client_id
+          xact_vol_ir_client_xact_id := io.irel().client_xact_id
+        }
+        // If this release has data, set all the pending bits except the first.
+        // Otherwise, clear all the pending bits
+        pending_irel_data := Mux(io.irel().hasMultibeatData(),
+                               dropPendingBitWhenBeatHasData(io.inner.release),
+                               UInt(0))
+      } .otherwise {
+        pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release))
+      }
+      if (p(EnableL2Logging)) {
+        when (io.irel().hasData()) {
+          printf("[release] addr_block=%x addr_beat=%d data=%x\n",
+                  io.irel().addr_block, io.irel().addr_beat, io.irel().data)
+        }
+      }
+    }
+
+    io.inner.grant.valid := state.isOneOf(s_wb_req, s_wb_resp, s_inner_probe, s_busy) &&
+                              vol_ignt_counter.pending &&
+                              !(pending_irel_data.orR || block_vol_ignt)
+
+    io.inner.grant.bits := inner_coh.makeGrant(xact_vol_irel)
+
+    scoreboard += (pending_irel_data.orR, vol_ignt_counter.pending)
+  }
+
+}
+
+trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
+  val pending_orel_send = Reg(init=Bool(false))
+  val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats))
+  val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus)
+  val pending_orel = pending_orel_send || pending_orel_data.orR || vol_ognt_counter.pending
+  val sending_orel = Reg(init = Bool(false))
+
+  // Block acceptance of inner releases if we have already started sending
+  // outer releases, but have not yet sent out the beat corresponding to the
+  // inner release. This function must be included in io.inner.release.ready
+  // if it is possible to start accepting a new inner release as the previous
+  // outer release is still being sent. DO NOT include this in the
+  // io.inner.release.ready if the releases are not buffered
+  // (i.e. io.inner.release and io.outer.release combinationally linked).
+  def blockInnerRelease(rel: ReleaseMetadata = io.irel()): Bool = {
+    val waiting_to_send = sending_orel && pending_orel_data(rel.addr_beat)
+    val sending_now = io.outer.release.fire() && rel.addr_beat === io.orel().addr_beat
+    rel.hasData() && (waiting_to_send || sending_now)
+  }
+
+  def outerRelease(
+      coh: ClientMetadata,
+      buffering: Bool = Bool(true),
+      data: UInt = io.irel().data,
+      add_pending_data_bits: UInt = UInt(0),
+      add_pending_send_bit: Bool = Bool(false),
+      block_orel: Bool = Bool(false)) {
+
+    when (state =/= s_idle || io.alloc.irel.should) {
+      pending_orel_data := (pending_orel_data |
+          addPendingBitWhenBeatHasData(io.inner.release) |
+          add_pending_data_bits) &
+        dropPendingBitWhenBeatHasData(io.outer.release)
+    }
+    when (add_pending_send_bit) { pending_orel_send := Bool(true) }
+    when (io.outer.release.fire()) {
+      when (io.outer.release.bits.first()) { sending_orel := Bool(true) }
+      when (io.outer.release.bits.last())  { sending_orel := Bool(false) }
+      pending_orel_send := Bool(false)
+    }
+
+    connectTwoWayBeatCounters(
+        status = vol_ognt_counter,
+        up = io.outer.release,
+        down = io.outer.grant,
+        trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(),
+        trackDown = (g: Grant) => g.isVoluntary())
+
+    io.outer.release.valid := !block_orel && Mux(buffering,
+      (state === s_busy) && Mux(io.orel().hasData(),
+        pending_orel_data(vol_ognt_counter.up.idx),
+        pending_orel_send),
+      // only writebacks need to be forwarded to the outer interface
+      state =/= s_idle && io.alloc.irel.matches &&
+        io.irel().hasData() && io.inner.release.valid)
+
+    io.outer.release.bits := coh.makeVoluntaryWriteback(
+      client_xact_id = UInt(0), // TODO was tracker id, but not needed?
+      addr_block = xact_addr_block,
+      addr_beat = vol_ognt_counter.up.idx,
+      data = data)
+
+    when (vol_ognt_counter.pending) { io.outer.grant.ready := Bool(true) }
+
+    scoreboard += (pending_orel, vol_ognt_counter.pending)
+  }
+}
+
+trait EmitsInnerProbes extends HasBlockAddressBuffer
+    with HasXactTrackerStates 
+    with HasPendingBitHelpers {
+  def io: HierarchicalXactTrackerIO
+
+  val needs_probes = (innerNCachingClients > 0)
+  val pending_iprbs = Reg(UInt(width = max(innerNCachingClients, 1)))
+  val curr_probe_dst = PriorityEncoder(pending_iprbs)
+
+  def full_representation: UInt
+  def initializeProbes() {
+    if (needs_probes)
+      pending_iprbs := full_representation & ~io.incoherent.asUInt
+    else
+      pending_iprbs := UInt(0)
+  }
+  def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
+                         !io.irel().isVoluntary() &&
+                         state === s_inner_probe 
+
+  def innerProbe(prb: Probe, next: UInt) {
+    if (needs_probes) {
+      val irel_counter = Wire(new TwoWayBeatCounterStatus)
+
+      pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe)
+      io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR
+      io.inner.probe.bits := prb
+
+      connectTwoWayBeatCounters(
+        status = irel_counter,
+        up = io.inner.probe,
+        down = io.inner.release,
+        max = innerNCachingClients,
+        trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary())
+
+      when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) {
+        state := next
+      }
+    } else {
+      when (state === s_inner_probe) { state := next }
+    }
+
+    //N.B. no pending bits added to scoreboard because all handled in s_inner_probe
+  }
+}
+
+trait RoutesInParent extends HasBlockAddressBuffer
+    with HasXactTrackerStates {
+  def io: HierarchicalXactTrackerIO
+  type AddrComparison = HasCacheBlockAddress => Bool
+  def exactAddrMatch(a: HasCacheBlockAddress): Bool = a.conflicts(xact_addr_block)
+  def routeInParent(iacqMatches: AddrComparison = exactAddrMatch,
+            irelMatches: AddrComparison = exactAddrMatch,
+            oprbMatches: AddrComparison = exactAddrMatch,
+            iacqCanAlloc: Bool = Bool(false),
+            irelCanAlloc: Bool = Bool(false),
+            oprbCanAlloc: Bool = Bool(false)) {
+    io.alloc.iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq())
+    io.alloc.irel.matches := (state =/= s_idle) && irelMatches(io.irel())
+    io.alloc.oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb())
+    io.alloc.iacq.can := state === s_idle && iacqCanAlloc
+    io.alloc.irel.can := state === s_idle && irelCanAlloc
+    io.alloc.oprb.can := state === s_idle && oprbCanAlloc
+    io.alloc.addr_block := xact_addr_block
+    io.alloc.idle := state === s_idle
+  }
+}
+
+trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer
+    with AcceptsVoluntaryReleases
+    with HasXactTrackerStates
+    with HasPendingBitHelpers {
+  def io: HierarchicalXactTrackerIO
+  def nSecondaryMisses: Int
+  def alwaysWriteFullBeat: Boolean
+  def inner_coh: ManagerMetadata
+  def trackerId: Int
+
+  // Secondary miss queue holds transaction metadata used to make grants
+  lazy val ignt_q = Module(new Queue(
+        new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })),
+        1 + nSecondaryMisses))
+
+  val pending_ignt = Wire(Bool())
+  val ignt_data_idx = Wire(UInt())
+  val ignt_data_done = Wire(Bool())
+  val ifin_counter = Wire(new TwoWayBeatCounterStatus)
+  val pending_put_data = Reg(init=Bits(0, width = innerDataBeats))
+  val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats))
+
+  def iacq_same_xact: Bool =
+    (xact_iacq.client_xact_id === io.iacq().client_xact_id) &&
+      (xact_iacq.client_id === io.iacq().client_id) &&
+      pending_ignt
+  def iacq_same_xact_multibeat = iacq_same_xact && io.iacq().hasMultibeatData()
+  def iacq_can_merge: Bool
+  def iacq_is_allocating: Bool = state === s_idle && io.alloc.iacq.should && io.inner.acquire.valid
+  def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid
+
+  def innerAcquire(can_alloc: Bool, next: UInt) {
+    val iacq_matches_head = iacq_same_xact && xact_iacq.addr_beat === io.iacq().addr_beat
+
+    // Enqueue some metadata information that we'll use to make coherence updates with later
+    ignt_q.io.enq.valid := iacq_is_allocating ||
+                           (!iacq_matches_head && pending_ignt &&
+                             io.inner.acquire.fire() && io.iacq().first())
+    ignt_q.io.enq.bits := io.iacq()
+
+    // Use the outputs of the queue to make further messages
+    xact_iacq := Mux(ignt_q.io.deq.valid, ignt_q.io.deq.bits, ignt_q.io.enq.bits)
+    xact_addr_beat := xact_iacq.addr_beat
+    pending_ignt := ignt_q.io.count > UInt(0)
+
+    // Track whether any beats are missing from a PutBlock
+    when (state =/= s_idle || io.alloc.iacq.should) {
+      pending_put_data := (pending_put_data &
+          dropPendingBitWhenBeatHasData(io.inner.acquire)) |
+          addPendingBitsOnFirstBeat(io.inner.acquire)
+    }
+
+    // Intialize transaction metadata for accepted Acquire
+    when(iacq_is_allocating) {
+      xact_addr_block := io.iacq().addr_block
+      xact_allocate := io.iacq().allocate() && can_alloc
+      xact_amo_shift_bytes := io.iacq().amo_shift_bytes()
+      xact_op_code := io.iacq().op_code()
+      xact_addr_byte := io.iacq().addr_byte()
+      xact_op_size := io.iacq().op_size()
+      // Make sure to collect all data from a PutBlock
+      pending_put_data := Mux(
+        io.iacq().isBuiltInType(Acquire.putBlockType),
+        dropPendingBitWhenBeatHasData(io.inner.acquire),
+        UInt(0))
+      pending_ignt_data := UInt(0)
+      state := next
+    }
+
+    scoreboard += (pending_put_data.orR)
+  }
+
+  def innerGrant(
+      data: UInt = io.ognt().data,
+      external_pending: Bool = Bool(false),
+      buffering: Bool = Bool(true),
+      add_pending_bits: UInt = UInt(0)) {
+    // Track the number of outstanding inner.finishes
+    connectTwoWayBeatCounters(
+      status = ifin_counter,
+      up = io.inner.grant,
+      down = io.inner.finish,
+      max = nSecondaryMisses,
+      trackUp = (g: Grant) => g.requiresAck())
+
+    // Track which beats are ready for response
+    when(!iacq_is_allocating) {
+      pending_ignt_data := pending_ignt_data |
+                             addPendingBitWhenBeatHasData(io.inner.release) |
+                             addPendingBitWhenBeatHasData(io.outer.grant) |
+                             add_pending_bits
+    }
+
+    if (p(EnableL2Logging)) {
+      when (io.inner.grant.fire() && io.ignt().hasData()) {
+        printf("[get] addr_block=%x addr_beat=%d data=%x\n",
+          xact_addr_block, io.ignt().addr_beat, io.ignt().data)
+      }
+    }
+
+    // Have we finished receiving the complete inner acquire transaction?
+    val iacq_finished = !(state === s_idle ||
+                          state === s_meta_read ||
+                          pending_put_data.orR)
+
+    val ignt_from_iacq = inner_coh.makeGrant(
+                            sec = ignt_q.io.deq.bits,
+                            manager_xact_id = UInt(trackerId), 
+                            data = data)
+
+    // Make the Grant message using the data stored in the secondary miss queue
+    val (cnt, done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat)
+    ignt_data_idx := cnt
+    ignt_data_done := done
+    ignt_q.io.deq.ready := Bool(false)
+    when(!vol_ignt_counter.pending) {
+      ignt_q.io.deq.ready := ignt_data_done
+      io.inner.grant.bits := ignt_from_iacq
+      io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter
+      when (state === s_busy && pending_ignt) {
+        io.inner.grant.valid := !external_pending && 
+          Mux(io.ignt().hasData(),
+            Mux(buffering,
+              pending_ignt_data(ignt_data_idx),
+              io.outer.grant.valid),
+            iacq_finished)
+      }
+    }
+
+    // We must wait for as many Finishes as we sent Grants
+    io.inner.finish.ready := state === s_busy
+
+    scoreboard += (pending_ignt, ifin_counter.pending)
+  }
+
+}
+
+trait EmitsOuterAcquires extends AcceptsInnerAcquires {
+  val ognt_counter = Wire(new TwoWayBeatCounterStatus)
+
+  // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory:
+  //
+  // If we're allocating in this cache, we can use the current metadata
+  // to make an appropriate custom Acquire, otherwise we copy over the
+  // built-in Acquire from the inner TL to the outer TL
+  def outerAcquire(
+      caching: Bool,
+      coh: ClientMetadata,
+      block_outer_acquire: Bool = Bool(false),
+      buffering: Bool = Bool(true),
+      data: UInt = io.iacq().data,
+      wmask: UInt = io.iacq().wmask(),
+      next: UInt = s_busy) {
+
+    // Tracks outstanding Acquires, waiting for their matching Grant.
+    connectTwoWayBeatCounters(
+      status = ognt_counter,
+      up = io.outer.acquire,
+      down = io.outer.grant,
+      beat = xact_addr_beat,
+      trackDown = (g: Grant) => !g.isVoluntary())
+
+    io.outer.acquire.valid :=
+      state === s_outer_acquire && !block_outer_acquire &&
+        (xact_allocate ||
+          Mux(buffering,
+            !pending_put_data(ognt_counter.up.idx),
+            // If not buffering, we should only send an outer acquire if
+            // the ignt_q is not empty (pending_ignt) and the enqueued
+            // transaction does not have data or we are receiving the
+            // inner acquire and it is the same transaction as the one enqueued.
+            pending_ignt && (!xact_iacq.hasData() ||
+              (io.inner.acquire.valid && iacq_same_xact))))
+
+    io.outer.acquire.bits :=
+      Mux(caching,
+        coh.makeAcquire(
+          op_code = xact_op_code,
+          client_xact_id = UInt(0),
+          addr_block = xact_addr_block),
+        BuiltInAcquireBuilder(
+          a_type = xact_iacq.a_type,
+          client_xact_id = UInt(0),
+          addr_block = xact_addr_block,
+          addr_beat = ognt_counter.up.idx,
+          data = data,
+          addr_byte = xact_addr_byte,
+          operand_size = xact_op_size,
+          opcode = xact_op_code,
+          wmask = wmask,
+          alloc = Bool(false))
+          (p.alterPartial({ case TLId => p(OuterTLId)})))
+
+    when(state === s_outer_acquire && ognt_counter.up.done) { state := next }
+
+    when (ognt_counter.pending) { io.outer.grant.ready := Bool(true) }
+
+    scoreboard += ognt_counter.pending
+  }
+}
+
+abstract class VoluntaryReleaseTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
+    with AcceptsVoluntaryReleases
+    with RoutesInParent {
+  def irel_can_merge = Bool(false)
+  def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
+                          io.irel().isVoluntary() &&
+                          pending_irel_data.orR
+}
+
+abstract class AcquireTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
+    with AcceptsInnerAcquires
+    with EmitsOuterAcquires
+    with EmitsInnerProbes
+    with RoutesInParent {
+}