refactor uncore files into separate packages

uncore/src/main/scala/agents/Agents.scala

@@ -0,0 +1,154 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import cde.{Parameters, Field}
+import junctions._
+import uncore.tilelink._
+import uncore.converters._
+import uncore.coherence._
+
+case object NReleaseTransactors extends Field[Int]
+case object NProbeTransactors extends Field[Int]
+case object NAcquireTransactors extends Field[Int]
+
+trait HasCoherenceAgentParameters {
+  implicit val p: Parameters
+  val nReleaseTransactors = 1
+  val nAcquireTransactors = p(NAcquireTransactors)
+  val nTransactors = nReleaseTransactors + nAcquireTransactors
+  val blockAddrBits = p(PAddrBits) - p(CacheBlockOffsetBits)
+  val outerTLId = p(OuterTLId)
+  val outerTLParams = p(TLKey(outerTLId))
+  val outerDataBeats = outerTLParams.dataBeats
+  val outerDataBits = outerTLParams.dataBitsPerBeat
+  val outerBeatAddrBits = log2Up(outerDataBeats)
+  val outerByteAddrBits = log2Up(outerDataBits/8)
+  val outerWriteMaskBits = outerTLParams.writeMaskBits
+  val innerTLId = p(InnerTLId)
+  val innerTLParams = p(TLKey(innerTLId))
+  val innerDataBeats = innerTLParams.dataBeats
+  val innerDataBits = innerTLParams.dataBitsPerBeat
+  val innerWriteMaskBits = innerTLParams.writeMaskBits
+  val innerBeatAddrBits = log2Up(innerDataBeats)
+  val innerByteAddrBits = log2Up(innerDataBits/8)
+  val innerNCachingClients = innerTLParams.nCachingClients
+  val maxManagerXacts = innerTLParams.maxManagerXacts
+  require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement
+}
+
+abstract class CoherenceAgentModule(implicit val p: Parameters) extends Module
+  with HasCoherenceAgentParameters
+abstract class CoherenceAgentBundle(implicit val p: Parameters) extends junctions.ParameterizedBundle()(p)
+   with HasCoherenceAgentParameters
+
+trait HasCoherenceAgentWiringHelpers {
+  def doOutputArbitration[T <: TileLinkChannel](
+      out: DecoupledIO[T],
+      ins: Seq[DecoupledIO[T]]) {
+    def lock(o: T) = o.hasMultibeatData()
+    val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, Some(lock _)))
+    out <> arb.io.out
+    arb.io.in <> ins
+  }
+
+  def doInputRouting[T <: Bundle with HasManagerTransactionId](
+        in: DecoupledIO[T],
+        outs: Seq[DecoupledIO[T]]) {
+    val idx = in.bits.manager_xact_id
+    outs.map(_.bits := in.bits)
+    outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) }
+    in.ready := Vec(outs.map(_.ready)).read(idx)
+  }
+
+  /** Broadcasts valid messages on this channel to all trackers,
+    * but includes logic to allocate a new tracker in the case where
+    * no previously allocated tracker matches the new req's addr.
+    *
+    * When a match is reported, if ready is high the new transaction
+    * is merged; when ready is low the transaction is being blocked.
+    * When no match is reported, any high idles are presumed to be
+    * from trackers that are available for allocation, and one is
+    * assigned via alloc based on priority; if no idles are high then
+    * all trackers are busy with other transactions. If idle is high
+    * but ready is low, the tracker will be allocated but does not
+    * have sufficient buffering for the data.
+    */ 
+  def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData](
+        in: DecoupledIO[T],
+        outs: Seq[DecoupledIO[T]],
+        allocs: Seq[TrackerAllocation],
+        dataOverrides: Option[Seq[UInt]] = None,
+        allocOverride: Option[Bool] = None,
+        matchOverride: Option[Bool] = None) {
+    val ready_bits = Vec(outs.map(_.ready)).toBits
+    val can_alloc_bits = Vec(allocs.map(_.can)).toBits
+    val should_alloc_bits = PriorityEncoderOH(can_alloc_bits)
+    val match_bits = Vec(allocs.map(_.matches)).toBits
+    val no_matches = !match_bits.orR
+    val alloc_ok = allocOverride.getOrElse(Bool(true))
+    val match_ok = matchOverride.getOrElse(Bool(true))
+    in.ready := (Mux(no_matches, can_alloc_bits, match_bits) & ready_bits).orR && alloc_ok && match_ok
+    outs.zip(allocs).zipWithIndex.foreach { case((out, alloc), i) =>
+      out.valid := in.valid && match_ok && alloc_ok
+      out.bits := in.bits
+      dataOverrides foreach { d => out.bits.data := d(i) }
+      alloc.should := should_alloc_bits(i) && no_matches && alloc_ok
+    }
+  }
+}
+
+trait HasInnerTLIO extends HasCoherenceAgentParameters {
+  val inner = new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)}))
+  val incoherent = Vec(inner.tlNCachingClients, Bool()).asInput
+  def iacq(dummy: Int = 0) = inner.acquire.bits
+  def iprb(dummy: Int = 0) = inner.probe.bits
+  def irel(dummy: Int = 0) = inner.release.bits
+  def ignt(dummy: Int = 0) = inner.grant.bits
+  def ifin(dummy: Int = 0) = inner.finish.bits
+}
+
+trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters {
+  val outer = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))
+  def oacq(dummy: Int = 0) = outer.acquire.bits
+  def ognt(dummy: Int = 0) = outer.grant.bits
+}
+
+trait HasCachedOuterTLIO extends HasCoherenceAgentParameters {
+  val outer = new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))
+  def oacq(dummy: Int = 0) = outer.acquire.bits
+  def oprb(dummy: Int = 0) = outer.probe.bits
+  def orel(dummy: Int = 0) = outer.release.bits
+  def ognt(dummy: Int = 0) = outer.grant.bits
+}
+
+class ManagerTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+  with HasInnerTLIO
+  with HasUncachedOuterTLIO
+
+abstract class CoherenceAgent(implicit p: Parameters) extends CoherenceAgentModule()(p) {
+  def innerTL: ManagerTileLinkIO
+  def outerTL: ClientTileLinkIO
+  def incoherent: Vec[Bool]
+}
+
+abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p)
+    with HasCoherenceAgentWiringHelpers {
+  val io = new ManagerTLIO
+  def innerTL = io.inner
+  def outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)}))
+  def incoherent = io.incoherent
+}
+
+class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+  with HasInnerTLIO
+  with HasCachedOuterTLIO
+
+abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p)
+    with HasCoherenceAgentWiringHelpers {
+  val io = new HierarchicalTLIO
+  def innerTL = io.inner
+  def outerTL = io.outer
+  def incoherent = io.incoherent
+}

uncore/src/main/scala/agents/Broadcast.scala

@@ -0,0 +1,202 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.constants._
+import cde.Parameters
+
+class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
+
+  // Create TSHRs for outstanding transactions
+  val irelTrackerList =
+    (0 until nReleaseTransactors).map(id =>
+      Module(new BufferedBroadcastVoluntaryReleaseTracker(id)))
+  val iacqTrackerList = 
+    (nReleaseTransactors until nTransactors).map(id =>
+      Module(new BufferedBroadcastAcquireTracker(id)))
+  val trackerList = irelTrackerList ++ iacqTrackerList
+
+  // Propagate incoherence flags
+  trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
+
+  // Create an arbiter for the one memory port
+  val outerList = trackerList.map(_.io.outer)
+  val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
+                                                    (p.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> outerList
+  io.outer <> outer_arb.io.out
+
+  // Handle acquire transaction initiation
+  val irel_vs_iacq_conflict =
+    io.inner.acquire.valid &&
+    io.inner.release.valid &&
+    io.irel().conflicts(io.iacq())
+
+  doInputRoutingWithAllocation(
+    in = io.inner.acquire,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc.iacq),
+    allocOverride = Some(!irel_vs_iacq_conflict))
+
+  // Handle releases, which might be voluntary and might have data
+  doInputRoutingWithAllocation(
+    in = io.inner.release,
+    outs = trackerList.map(_.io.inner.release),
+    allocs = trackerList.map(_.io.alloc.irel))
+
+  // Wire probe requests and grant reply to clients, finish acks from clients
+  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
+
+  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
+
+  doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
+}
+
+class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+}
+
+trait BroadcastsToAllClients extends HasCoherenceAgentParameters {
+  val coh = HierarchicalMetadata.onReset
+  val inner_coh = coh.inner
+  val outer_coh = coh.outer
+  def full_representation = ~UInt(0, width = innerNCachingClients)
+}
+
+abstract class BroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends VoluntaryReleaseTracker(trackerId)(p) 
+    with EmitsVoluntaryReleases
+    with BroadcastsToAllClients {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+
+  // Checks for illegal behavior
+  assert(!(state === s_idle && io.inner.release.fire() && io.alloc.irel.should && !io.irel().isVoluntary()),
+    "VoluntaryReleaseTracker accepted Release that wasn't voluntary!")
+}
+
+abstract class BroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends AcquireTracker(trackerId)(p) 
+    with EmitsVoluntaryReleases
+    with BroadcastsToAllClients {
+  val io = new HierarchicalXactTrackerIO
+  pinAllReadyValidLow(io)
+
+  val alwaysWriteFullBeat = false
+  val nSecondaryMisses = 1
+  def iacq_can_merge = Bool(false)
+
+  // Checks for illegal behavior
+  // TODO: this could be allowed, but is a useful check against allocation gone wild
+  assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq.should &&
+    io.iacq().hasMultibeatData() && !io.iacq().first()),
+    "AcquireTracker initialized with a tail data beat.")
+
+  assert(!(state =/= s_idle && pending_ignt && xact_iacq.isPrefetch()),
+    "Broadcast Hub does not support Prefetches.")
+
+  assert(!(state =/= s_idle && pending_ignt && xact_iacq.isAtomic()),
+    "Broadcast Hub does not support PutAtomics.")
+}
+
+class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastVoluntaryReleaseTracker(trackerId)(p)
+    with HasDataBuffer {
+
+  // Tell the parent if any incoming messages conflict with the ongoing transaction
+  routeInParent()
+  io.alloc.iacq.can := Bool(false)
+
+  // Start transaction by accepting inner release
+  innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
+
+  // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
+  io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact
+
+  when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data }
+
+  // Dispatch outer release
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    data = data_buffer(vol_ognt_counter.up.idx),
+    add_pending_send_bit = irel_is_allocating)
+
+  quiesce() {}
+}
+
+class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastAcquireTracker(trackerId)(p)
+    with HasByteWriteMaskBuffer {
+
+  // Setup IOs used for routing in the parent
+  routeInParent()
+  io.alloc.irel.can := Bool(false)
+
+  // First, take care of accpeting new acquires or secondary misses
+  // Handling of primary and secondary misses' data and write mask merging
+  innerAcquire(
+    can_alloc = Bool(false),
+    next = s_inner_probe)
+
+  io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact 
+
+  // Track which clients yet need to be probed and make Probe message
+  // If a writeback occurs, we can forward its data via the buffer,
+  // and skip having to go outwards
+  val skip_outer_acquire = pending_ignt_data.andR
+
+  innerProbe(
+    inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
+    Mux(!skip_outer_acquire, s_outer_acquire, s_busy))
+
+  // Handle incoming releases from clients, which may reduce sharer counts
+  // and/or write back dirty data, and may be unexpected voluntary releases
+  def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
+                         io.irel().isVoluntary() &&
+                         !Vec(s_idle, s_meta_write).contains(state) &&
+                         !all_pending_done &&
+                         !io.outer.grant.fire() &&
+                         !io.inner.grant.fire() &&
+                         !vol_ignt_counter.pending
+
+  innerRelease(block_vol_ignt = vol_ognt_counter.pending) 
+
+  //TODO: accept vol irels when state === s_idle, operate like the VolRelTracker
+  io.inner.release.ready := irel_can_merge || irel_same_xact
+
+  mergeDataInner(io.inner.release)
+
+  // If there was a writeback, forward it outwards
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    data = data_buffer(vol_ognt_counter.up.idx))
+
+  // Send outer request for miss
+  outerAcquire(
+    caching = !xact_iacq.isBuiltInType(),
+    coh = outer_coh,
+    data = data_buffer(ognt_counter.up.idx),
+    wmask = wmask_buffer(ognt_counter.up.idx),
+    next = s_busy)
+    
+  // Handle the response from outer memory
+  mergeDataOuter(io.outer.grant)
+
+  // Acknowledge or respond with data
+  innerGrant(
+    data = data_buffer(ignt_data_idx),
+    external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending)
+
+  when(iacq_is_allocating) {
+    initializeProbes()
+  }
+
+  initDataInner(io.inner.acquire, iacq_is_allocating || iacq_is_merging)
+
+  // Wait for everything to quiesce
+  quiesce() { clearWmaskBuffer() }
+}

uncore/src/main/scala/agents/Bufferless.scala

@@ -0,0 +1,145 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.constants._
+import cde.Parameters
+
+
+class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) {
+
+  // Create TSHRs for outstanding transactions
+  val irelTrackerList =
+    (0 until nReleaseTransactors).map(id =>
+      Module(new BufferlessBroadcastVoluntaryReleaseTracker(id)))
+  val iacqTrackerList = 
+    (nReleaseTransactors until nTransactors).map(id =>
+      Module(new BufferlessBroadcastAcquireTracker(id)))
+  val trackerList = irelTrackerList ++ iacqTrackerList
+
+  // Propagate incoherence flags
+  trackerList.map(_.io.incoherent) foreach { _ := io.incoherent }
+
+  // Create an arbiter for the one memory port
+  val outerList = trackerList.map(_.io.outer)
+  val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size)
+                                                    (p.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> outerList
+  io.outer <> outer_arb.io.out
+
+  // Handle acquire transaction initiation
+  val irel_vs_iacq_conflict =
+    io.inner.acquire.valid &&
+    io.inner.release.valid &&
+    io.irel().conflicts(io.iacq())
+
+  doInputRoutingWithAllocation(
+    in = io.inner.acquire,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc.iacq),
+    allocOverride = Some(!irel_vs_iacq_conflict))
+  io.outer.acquire.bits.data := io.inner.acquire.bits.data
+  io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat
+
+  // Handle releases, which might be voluntary and might have data
+  doInputRoutingWithAllocation(
+    in = io.inner.release,
+    outs = trackerList.map(_.io.inner.release),
+    allocs = trackerList.map(_.io.alloc.irel))
+  io.outer.release.bits.data := io.inner.release.bits.data
+  io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat
+
+  // Wire probe requests and grant reply to clients, finish acks from clients
+  doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe))
+
+  doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant))
+  io.inner.grant.bits.data := io.outer.grant.bits.data
+  io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat
+
+  doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish))
+}
+
+class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastVoluntaryReleaseTracker(trackerId)(p) {
+
+  // Tell the parent if any incoming messages conflict with the ongoing transaction
+  routeInParent()
+  io.alloc.iacq.can := Bool(false)
+
+  // Start transaction by accepting inner release
+  innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending)
+
+  // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat
+  // and if the outer relase path is clear 
+  val irel_could_accept = state === s_idle || irel_can_merge || irel_same_xact
+  io.inner.release.ready := irel_could_accept &&
+    (!io.irel().hasData() || io.outer.release.ready)
+
+  // Dispatch outer release
+  outerRelease(coh = outer_coh.onHit(M_XWR))
+  io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data
+
+  quiesce() {}
+}
+
+class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters)
+    extends BroadcastAcquireTracker(trackerId)(p) {
+
+  // Setup IOs used for routing in the parent
+  routeInParent()
+  io.alloc.irel.can := Bool(false)
+
+  // First, take care of accpeting new acquires or secondary misses
+  // Handling of primary and secondary misses' data and write mask merging
+  innerAcquire(
+    can_alloc = Bool(false),
+    next = s_inner_probe)
+
+  val iacq_could_accept = state === s_outer_acquire || iacq_can_merge || iacq_same_xact
+  io.inner.acquire.ready := iacq_could_accept && 
+    (!io.iacq().hasData() || io.outer.acquire.fire())
+
+  // Track which clients yet need to be probed and make Probe message
+  innerProbe(
+    inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block),
+    s_outer_acquire)
+
+  // Handle incoming releases from clients, which may reduce sharer counts
+  // and/or write back dirty data, and may be unexpected voluntary releases
+  def irel_can_merge = io.irel().conflicts(xact_addr_block) &&
+                         io.irel().isVoluntary() &&
+                         !vol_ignt_counter.pending &&
+                         (state =/= s_idle)
+
+  innerRelease(block_vol_ignt = vol_ognt_counter.pending) 
+
+  val irel_could_accept = irel_can_merge || irel_same_xact
+  io.inner.release.ready := irel_could_accept &&
+    (!io.irel().hasData() || io.outer.release.ready)
+
+  // If there was a writeback, forward it outwards
+  outerRelease(
+    coh = outer_coh.onHit(M_XWR),
+    buffering = Bool(false))
+
+  // Send outer request for miss
+  outerAcquire(
+    caching = !xact_iacq.isBuiltInType(),
+    buffering = Bool(false),
+    coh = outer_coh,
+    next = s_busy)
+
+  // Handle the response from outer memory
+  io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data
+
+  // Acknowledge or respond with data
+  innerGrant(external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending)
+
+  when(iacq_is_allocating) { initializeProbes() }
+
+  // Wait for everything to quiesce
+  quiesce() {}
+}

uncore/src/main/scala/agents/Ecc.scala

@@ -0,0 +1,146 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+
+abstract class Decoding
+{
+  def uncorrected: UInt
+  def corrected: UInt
+  def correctable: Bool
+  def uncorrectable: Bool
+  def error = correctable || uncorrectable
+}
+
+abstract class Code
+{
+  def width(w0: Int): Int
+  def encode(x: UInt): UInt
+  def decode(x: UInt): Decoding
+}
+
+class IdentityCode extends Code
+{
+  def width(w0: Int) = w0
+  def encode(x: UInt) = x
+  def decode(y: UInt) = new Decoding {
+    def uncorrected = y
+    def corrected = y
+    def correctable = Bool(false)
+    def uncorrectable = Bool(false)
+  }
+}
+
+class ParityCode extends Code
+{
+  def width(w0: Int) = w0+1
+  def encode(x: UInt) = Cat(x.xorR, x)
+  def decode(y: UInt) = new Decoding {
+    def uncorrected = y(y.getWidth-2,0)
+    def corrected = uncorrected
+    def correctable = Bool(false)
+    def uncorrectable = y.xorR
+  }
+}
+
+class SECCode extends Code
+{
+  def width(k: Int) = {
+    val m = log2Floor(k) + 1
+    k + m + (if((1 << m) < m+k+1) 1 else 0)
+  }
+  def encode(x: UInt) = {
+    val k = x.getWidth
+    require(k > 0)
+    val n = width(k)
+
+    val y = for (i <- 1 to n) yield {
+      if (isPow2(i)) {
+        val r = for (j <- 1 to n; if j != i && (j & i) != 0)
+          yield x(mapping(j))
+        r reduce (_^_)
+      } else
+        x(mapping(i))
+    }
+    Vec(y).toBits
+  }
+  def decode(y: UInt) = new Decoding {
+    val n = y.getWidth
+    require(n > 0 && !isPow2(n))
+
+    val p2 = for (i <- 0 until log2Up(n)) yield 1 << i
+    val syndrome = p2 map { i =>
+      val r = for (j <- 1 to n; if (j & i) != 0)
+        yield y(j-1)
+      r reduce (_^_)
+    }
+    val s = Vec(syndrome).toBits
+
+    private def swizzle(z: UInt) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits
+    def uncorrected = swizzle(y)
+    def corrected = swizzle(((y.toUInt << 1) ^ UIntToOH(s)) >> 1)
+    def correctable = s.orR
+    def uncorrectable = Bool(false)
+  }
+  private def mapping(i: Int) = i-1-log2Up(i)
+}
+
+class SECDEDCode extends Code
+{
+  private val sec = new SECCode
+  private val par = new ParityCode
+
+  def width(k: Int) = sec.width(k)+1
+  def encode(x: UInt) = par.encode(sec.encode(x))
+  def decode(x: UInt) = new Decoding {
+    val secdec = sec.decode(x(x.getWidth-2,0))
+    val pardec = par.decode(x)
+
+    def uncorrected = secdec.uncorrected
+    def corrected = secdec.corrected
+    def correctable = pardec.uncorrectable
+    def uncorrectable = !pardec.uncorrectable && secdec.correctable
+  }
+}
+
+object ErrGen
+{
+  // generate a 1-bit error with approximate probability 2^-f
+  def apply(width: Int, f: Int): UInt = {
+    require(width > 0 && f >= 0 && log2Up(width) + f <= 16)
+    UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0)
+  }
+  def apply(x: UInt, f: Int): UInt = x ^ apply(x.getWidth, f)
+}
+
+class SECDEDTest extends Module
+{
+  val code = new SECDEDCode
+  val k = 4
+  val n = code.width(k)
+
+  val io = new Bundle {
+    val original = Bits(OUTPUT, k)
+    val encoded = Bits(OUTPUT, n)
+    val injected = Bits(OUTPUT, n)
+    val uncorrected = Bits(OUTPUT, k)
+    val corrected = Bits(OUTPUT, k)
+    val correctable = Bool(OUTPUT)
+    val uncorrectable = Bool(OUTPUT)
+  }
+
+  val c = Counter(Bool(true), 1 << k)
+  val numErrors = Counter(c._2, 3)._1
+  val e = code.encode(c._1)
+  val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1))
+  val d = code.decode(i)
+
+  io.original := c._1
+  io.encoded := e
+  io.injected := i
+  io.uncorrected := d.uncorrected
+  io.corrected := d.corrected
+  io.correctable := d.correctable
+  io.uncorrectable := d.uncorrectable
+}

uncore/src/main/scala/agents/Mmio.scala

@@ -0,0 +1,73 @@
+package uncore.agents
+
+import Chisel._
+import uncore.tilelink._
+import cde.Parameters
+
+class MMIOTileLinkManagerData(implicit p: Parameters)
+    extends TLBundle()(p)
+    with HasClientId
+    with HasClientTransactionId
+
+class MMIOTileLinkManager(implicit p: Parameters)
+    extends CoherenceAgentModule()(p) {
+  val io = new ManagerTLIO
+
+  // MMIO requests should never need probe or release
+  io.inner.probe.valid := Bool(false)
+  io.inner.release.ready := Bool(false)
+
+  val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData()
+  val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0)
+  val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1)
+
+  // Acquire and Grant are basically passthru,
+  // except client_id and client_xact_id need to be converted.
+  // Associate the inner client_id and client_xact_id
+  // with the outer client_xact_id.
+  val xact_pending = Reg(init = UInt(0, maxManagerXacts))
+  val xact_id_sel = PriorityEncoder(~xact_pending)
+  val xact_id_reg = RegEnable(xact_id_sel, multibeat_start)
+  val xact_multibeat = Reg(init = Bool(false))
+  val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel)
+  val xact_free = !xact_pending.andR
+  val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData))
+
+  io.inner.acquire.ready := io.outer.acquire.ready && xact_free
+  io.outer.acquire.valid := io.inner.acquire.valid && xact_free
+  io.outer.acquire.bits  := io.inner.acquire.bits
+  io.outer.acquire.bits.client_xact_id := outer_xact_id
+
+  def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool =
+    !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1)
+
+  def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt =
+    Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0))
+
+  def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt =
+    ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(),
+      UIntToOH(in.bits.manager_xact_id), UInt(0))
+
+  def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt =
+    ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0))
+
+  xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) &
+                                  clearPendingBitOnFin(io.inner.finish) &
+                                  clearPendingBitOnGnt(io.inner.grant)
+
+  when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) {
+    xact_buffer(outer_xact_id) := io.iacq()
+  }
+
+  when (multibeat_start) { xact_multibeat := Bool(true) }
+  when (multibeat_end)   { xact_multibeat := Bool(false) }
+
+  val gnt_xact = xact_buffer(io.ognt().client_xact_id)
+  io.outer.grant.ready := io.inner.grant.ready
+  io.inner.grant.valid := io.outer.grant.valid
+  io.inner.grant.bits  := io.outer.grant.bits
+  io.inner.grant.bits.client_id := gnt_xact.client_id
+  io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id
+  io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id
+  io.inner.finish.ready := Bool(true)
+}

uncore/src/main/scala/agents/StoreDataQueue.scala

@@ -0,0 +1,119 @@
+// See LICENSE for license details.
+
+package uncore.agents
+import Chisel._
+import uncore.tilelink._
+import cde.{Parameters, Field}
+
+case object L2StoreDataQueueDepth extends Field[Int]
+
+trait HasStoreDataQueueParameters extends HasCoherenceAgentParameters {
+  val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats
+  val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth))
+  val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases
+}
+
+class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p)
+    with HasStoreDataQueueParameters {
+  val idx = UInt(width = dqIdxBits)
+  val loc = UInt(width = log2Ceil(nDataQueueLocations))
+} 
+
+object DataQueueLocation {
+  def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = {
+    val d = Wire(new DataQueueLocation)
+    d.idx := idx
+    d.loc := loc
+    d
+  }
+}
+
+trait HasStoreDataQueue extends HasStoreDataQueueParameters {
+  val io: HierarchicalTLIO
+  val trackerIOsList: Seq[HierarchicalXactTrackerIO]
+
+  val internalDataBits = new DataQueueLocation().getWidth
+  val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations)
+
+  val usingStoreDataQueue = p.alterPartial({
+    case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
+    case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits))
+  })
+
+  // Queue to store impending Put data
+  lazy val sdq = Reg(Vec(sdqDepth, io.iacq().data))
+  lazy val sdq_val = Reg(init=Bits(0, sdqDepth))
+  lazy val sdq_alloc_id = PriorityEncoder(~sdq_val)
+  lazy val sdq_rdy = !sdq_val.andR
+  lazy val sdq_enq = trackerIOsList.map( t =>
+                  (t.alloc.iacq.should || t.alloc.iacq.matches) &&
+                    t.inner.acquire.fire() &&
+                    t.iacq().hasData()
+                ).reduce(_||_)
+
+  lazy val sdqLoc = List.fill(nTransactors) {
+    DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits
+  }
+
+  /*
+  doInputRoutingWithAllocation(
+    in = io.inner.acquire,
+    outs = trackerList.map(_.io.inner.acquire),
+    allocs = trackerList.map(_.io.alloc._iacq),
+    dataOverride = Some(sdqLoc),
+    allocOverride = Some(sdq_rdy && !irel_vs_iacq_conflict))
+  */
+
+  // Queue to store impending Voluntary Release data
+  lazy val voluntary = io.irel().isVoluntary()
+  lazy val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData()
+  lazy val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width
+  lazy val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1
+  
+
+  lazy val vwbqLoc = (0 until nTransactors).map(i =>
+    (DataQueueLocation(rel_data_cnt,
+                       (if(i < nReleaseTransactors) inVolWBQueue
+                        else inClientReleaseQueue)).toBits))
+  /*
+  doInputRoutingWithAllocation(
+    io.inner.release,
+    trackerList.map(_.io.inner.release),
+    trackerList.map(_.io.matches.irel),
+    trackerList.map(_.io.alloc.irel),
+    Some(vwbqLoc))
+  */
+
+  val outer_arb: ClientTileLinkIOArbiter
+  lazy val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data)
+  /*
+  val outer_arb = Module(new ClientTileLinkIOArbiter(trackerList.size)
+                    (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) })))
+  outer_arb.io.in <> trackerList
+  */
+  // Get the pending data out of the store data queue
+  lazy val is_in_sdq = outer_data_ptr.loc === inStoreQueue
+  lazy val free_sdq = io.outer.acquire.fire() &&
+                  io.outer.acquire.bits.hasData() &&
+                  outer_data_ptr.loc === inStoreQueue
+  /*
+  io.outer <> outer_arb.io.out
+  io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array(
+                                          inStoreQueue -> sdq(outer_data_ptr.idx),
+                                          inVolWBQueue -> vwbdq(outer_data_ptr.idx)))
+  */
+
+  // Enqueue SDQ data
+  def sdqEnqueue() {
+    when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data }
+    when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data }
+  }
+
+  // Update SDQ valid bits
+  def sdqUpdate() {
+    when (io.outer.acquire.valid || sdq_enq) {
+      sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | 
+                 PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
+    }
+  }
+}

uncore/src/main/scala/agents/Trackers.scala

@@ -0,0 +1,582 @@
+// See LICENSE for license details.
+
+package uncore.agents
+
+import Chisel._
+import uncore.coherence._
+import uncore.tilelink._
+import uncore.util._
+import cde.Parameters
+import scala.math.max
+
+class TrackerAllocation extends Bundle {
+  val matches = Bool(OUTPUT)
+  val can = Bool(OUTPUT)
+  val should = Bool(INPUT)
+}
+
+trait HasTrackerAllocationIO extends Bundle {
+  val alloc = new Bundle {
+    val iacq = new TrackerAllocation
+    val irel = new TrackerAllocation
+    val oprb = new TrackerAllocation
+  }
+}
+
+class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p)
+  with HasTrackerAllocationIO
+
+class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p)
+  with HasTrackerAllocationIO
+
+abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p)
+    with HasXactTrackerStates
+    with HasPendingBitHelpers {
+  override val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9)
+  val state = Reg(init=s_idle)
+
+  def quiesce(next: UInt = s_idle)(restore: => Unit) {
+    all_pending_done := !scoreboard.foldLeft(Bool(false))(_||_)
+    when(state === s_busy && all_pending_done) {
+      state := next
+      restore
+    }
+  }
+
+  def pinAllReadyValidLow[T <: Data](b: Bundle) {
+    b.elements.foreach {
+      _._2 match {
+        case d: DecoupledIO[_] =>
+          if(d.ready.dir == OUTPUT) d.ready := Bool(false)
+          else if(d.valid.dir == OUTPUT) d.valid := Bool(false)
+        case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) 
+        case b: Bundle => pinAllReadyValidLow(b)
+        case _ =>
+      }
+    }
+  }
+}
+
+trait HasXactTrackerStates {
+  def state: UInt 
+  def s_idle: UInt = UInt(0)
+  def s_meta_read: UInt
+  def s_meta_resp: UInt
+  def s_wb_req: UInt
+  def s_wb_resp: UInt
+  def s_inner_probe: UInt
+  def s_outer_acquire: UInt
+  def s_busy: UInt
+  def s_meta_write: UInt
+}
+
+trait HasPendingBitHelpers extends HasDataBeatCounters {
+  val scoreboard = scala.collection.mutable.ListBuffer.empty[Bool]
+  val all_pending_done = Wire(Bool())
+
+  def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt =
+    Fill(in.tlDataBeats, inc) &  UIntToOH(in.addr_beat)
+
+  def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt =
+    ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat)
+
+  def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt =
+    Fill(in.tlNCachingClients, inc) &  UIntToOH(in.client_id)
+
+  def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt =
+    ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id)
+
+  def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt =
+    addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits)
+
+  def addPendingBitWhenBeatHasDataAndAllocs(
+      in: DecoupledIO[AcquireFromSrc],
+      alloc_override: Bool = Bool(false)): UInt =
+    addPendingBitWhenBeatHasData(in, in.bits.allocate() || alloc_override)
+
+  def addPendingBitWhenBeatNeedsRead(in: DecoupledIO[AcquireFromSrc], inc: Bool = Bool(true)): UInt = {
+    val a = in.bits
+    val needs_read = (a.isGet() || a.isAtomic() || a.hasPartialWritemask()) || inc
+    addPendingBitWhenBeat(in.fire() && needs_read, a)
+  }
+
+  def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt =
+    addPendingBitWhenBeat(in.fire() && in.bits.hasPartialWritemask(), in.bits)
+
+  def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt =
+    Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat))
+
+  def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits)
+
+  def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenId(in.fire(), in.bits)
+
+  def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
+    dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
+
+  def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt =
+    addPendingBitWhenId(in.fire(), in.bits)
+
+  def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt =
+    addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits)
+
+  def addOtherBits(en: Bool, nBits: Int): UInt =
+    Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits))
+
+  def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
+    addOtherBits(in.fire() &&
+                 in.bits.hasMultibeatData() &&
+                 in.bits.addr_beat === UInt(0),
+                 in.bits.tlDataBeats)
+
+  def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt =
+    ~addPendingBitsOnFirstBeat(in)
+}
+
+trait HasDataBuffer extends HasCoherenceAgentParameters {
+  val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits)))
+
+  type TLDataBundle = TLBundle with HasTileLinkData with HasTileLinkBeatId
+
+  def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) {
+    when(in.fire() && in.bits.hasData() && alloc) { 
+      data_buffer(in.bits.addr_beat) := in.bits.data
+    }
+  }
+
+  // TODO: provide func for accessing when innerDataBeats =/= outerDataBeats or internalDataBeats
+  def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
+    data_buffer(beat) := incoming 
+  }
+
+  def mergeDataInner[T <: TLDataBundle](in: DecoupledIO[T]) {
+    when(in.fire() && in.bits.hasData()) { 
+      mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data)
+    }
+  }
+
+  def mergeDataOuter[T <: TLDataBundle](in: DecoupledIO[T]) {
+    when(in.fire() && in.bits.hasData()) { 
+      mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data)
+    }
+  }
+}
+
+trait HasByteWriteMaskBuffer extends HasDataBuffer { 
+  val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerWriteMaskBits)))
+
+  override def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) {
+    when(in.fire() && in.bits.hasData() && alloc) { 
+      val beat = in.bits.addr_beat
+      val full = FillInterleaved(8, in.bits.wmask())
+      data_buffer(beat) := (~full & data_buffer(beat)) | (full & in.bits.data)
+      wmask_buffer(beat) := in.bits.wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed
+    }
+  }
+
+  override def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) {
+    val old_data = incoming     // Refilled, written back, or de-cached data
+    val new_data = data_buffer(beat) // Newly Put data is already in the buffer
+    val wmask = FillInterleaved(8, wmask_buffer(beat))
+    data_buffer(beat) := ~wmask & old_data | wmask & new_data
+  }
+
+  def clearWmaskBuffer() {
+    wmask_buffer.foreach { w => w := UInt(0) }
+  }
+}
+
+trait HasBlockAddressBuffer extends HasCoherenceAgentParameters {
+  val xact_addr_block = Reg(init = UInt(0, width = blockAddrBits))
+}
+
+
+trait HasAcquireMetadataBuffer extends HasBlockAddressBuffer {
+  val xact_allocate = Reg{ Bool() }
+  val xact_amo_shift_bytes = Reg{ UInt() }
+  val xact_op_code = Reg{ UInt() }
+  val xact_addr_byte = Reg{ UInt() }
+  val xact_op_size = Reg{ UInt() }
+  val xact_addr_beat = Wire(UInt())
+  val xact_iacq = Wire(new SecondaryMissInfo)
+}
+
+trait HasVoluntaryReleaseMetadataBuffer extends HasBlockAddressBuffer
+    with HasPendingBitHelpers
+    with HasXactTrackerStates {
+  def io: HierarchicalXactTrackerIO
+
+  val xact_vol_ir_r_type = Reg{ UInt() }
+  val xact_vol_ir_src = Reg{ UInt() }
+  val xact_vol_ir_client_xact_id = Reg{ UInt() }
+
+  def xact_vol_irel = Release(
+                        src = xact_vol_ir_src,
+                        voluntary = Bool(true),
+                        r_type = xact_vol_ir_r_type,
+                        client_xact_id = xact_vol_ir_client_xact_id,
+                        addr_block = xact_addr_block)
+                        (p.alterPartial({ case TLId => p(InnerTLId) }))
+}
+
+trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
+  def inner_coh: ManagerMetadata
+
+  val pending_irel_data = Reg(init=Bits(0, width = innerDataBeats))
+  val vol_ignt_counter = Wire(new TwoWayBeatCounterStatus)
+
+  def irel_can_merge: Bool
+  def irel_same_xact: Bool
+  def irel_is_allocating: Bool = state === s_idle && io.alloc.irel.should && io.inner.release.valid
+  def irel_is_merging: Bool = (irel_can_merge || irel_same_xact) && io.inner.release.valid
+
+  def innerRelease(block_vol_ignt: Bool = Bool(false), next: UInt = s_busy) {
+    connectTwoWayBeatCounters(
+      status = vol_ignt_counter,
+      up = io.inner.release,
+      down = io.inner.grant,
+      trackUp = (r: Release) => {
+        Mux(state === s_idle, io.alloc.irel.should, io.alloc.irel.matches) && r.isVoluntary() && r.requiresAck()
+      },
+      trackDown = (g: Grant) => (state =/= s_idle) && g.isVoluntary())
+
+
+    when(irel_is_allocating) {
+      xact_addr_block := io.irel().addr_block
+      state := next
+    }
+
+    when(io.inner.release.fire()) {
+      when(io.alloc.irel.should || (irel_can_merge && io.irel().first())) {
+        xact_vol_ir_r_type := io.irel().r_type
+        xact_vol_ir_src := io.irel().client_id
+        xact_vol_ir_client_xact_id := io.irel().client_xact_id
+        // If this release has data, set all the pending bits except the first.
+        // Otherwise, clear all the pending bits
+        pending_irel_data := Mux(io.irel().hasMultibeatData(),
+                               dropPendingBitWhenBeatHasData(io.inner.release),
+                               UInt(0))
+      } .elsewhen (irel_same_xact) {
+        pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release))
+      }
+    }
+
+    io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) &&
+                              vol_ignt_counter.pending &&
+                              !(pending_irel_data.orR || block_vol_ignt)
+
+    io.inner.grant.bits := inner_coh.makeGrant(xact_vol_irel)
+
+    scoreboard += (pending_irel_data.orR, vol_ignt_counter.pending)
+  }
+
+}
+
+trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer {
+  val pending_orel_send = Reg(init=Bool(false))
+  val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats))
+  val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus)
+  val pending_orel = pending_orel_send || pending_orel_data.orR || vol_ognt_counter.pending
+
+  def outerRelease(
+      coh: ClientMetadata,
+      buffering: Bool = Bool(true),
+      data: UInt = io.irel().data,
+      add_pending_data_bits: UInt = UInt(0),
+      add_pending_send_bit: Bool = Bool(false)) {
+
+    when (state =/= s_idle || io.alloc.irel.should) {
+      pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) |
+                            addPendingBitWhenBeatHasData(io.inner.release) |
+                            add_pending_data_bits
+    }
+    when (add_pending_send_bit) { pending_orel_send := Bool(true) }
+    when (io.outer.release.fire()) { pending_orel_send := Bool(false) }
+
+    connectTwoWayBeatCounters(
+        status = vol_ognt_counter,
+        up = io.outer.release,
+        down = io.outer.grant,
+        trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(),
+        trackDown = (g: Grant) => g.isVoluntary())
+
+    io.outer.release.valid := state === s_busy &&
+                                Mux(io.orel().hasData(),
+                                  Mux(buffering, 
+                                      pending_orel_data(vol_ognt_counter.up.idx),
+                                      io.inner.release.valid),
+                                  pending_orel)
+
+
+    io.outer.release.bits := coh.makeVoluntaryWriteback(
+      client_xact_id = UInt(0), // TODO was tracker id, but not needed?
+      addr_block = xact_addr_block,
+      addr_beat = vol_ognt_counter.up.idx,
+      data = data)
+
+    io.outer.grant.ready := state === s_busy
+
+    scoreboard += (pending_orel, vol_ognt_counter.pending)
+  }
+}
+
+trait EmitsInnerProbes extends HasBlockAddressBuffer
+    with HasXactTrackerStates 
+    with HasPendingBitHelpers {
+  def io: HierarchicalXactTrackerIO
+
+  val needs_probes = (innerNCachingClients > 0)
+  val pending_iprbs = Reg(UInt(width = max(innerNCachingClients, 1)))
+  val curr_probe_dst = PriorityEncoder(pending_iprbs)
+
+  def full_representation: UInt
+  def initializeProbes() {
+    if (needs_probes)
+      pending_iprbs := full_representation & ~io.incoherent.toBits
+    else
+      pending_iprbs := UInt(0)
+  }
+  def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
+                         !io.irel().isVoluntary() &&
+                         state === s_inner_probe 
+
+  def innerProbe(prb: Probe, next: UInt) {
+    if (needs_probes) {
+      val irel_counter = Wire(new TwoWayBeatCounterStatus)
+
+      pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe)
+      io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR
+      io.inner.probe.bits := prb
+
+      connectTwoWayBeatCounters(
+        status = irel_counter,
+        up = io.inner.probe,
+        down = io.inner.release,
+        max = innerNCachingClients,
+        trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary())
+
+      when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) {
+        state := next
+      }
+    } else {
+      when (state === s_inner_probe) { state := next }
+    }
+
+    //N.B. no pending bits added to scoreboard because all handled in s_inner_probe
+  }
+}
+
+trait RoutesInParent extends HasBlockAddressBuffer
+    with HasXactTrackerStates {
+  def io: HierarchicalXactTrackerIO
+  type AddrComparison = HasCacheBlockAddress => Bool
+  def exactAddrMatch(a: HasCacheBlockAddress): Bool = a.conflicts(xact_addr_block)
+  def routeInParent(iacqMatches: AddrComparison = exactAddrMatch,
+            irelMatches: AddrComparison = exactAddrMatch,
+            oprbMatches: AddrComparison = exactAddrMatch) {
+    io.alloc.iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq())
+    io.alloc.irel.matches := (state =/= s_idle) && irelMatches(io.irel())
+    io.alloc.oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb())
+    io.alloc.iacq.can := state === s_idle
+    io.alloc.irel.can := state === s_idle
+    io.alloc.oprb.can := Bool(false)
+  }
+}
+
+trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer
+    with AcceptsVoluntaryReleases
+    with HasXactTrackerStates
+    with HasPendingBitHelpers {
+  def io: HierarchicalXactTrackerIO
+  def nSecondaryMisses: Int
+  def alwaysWriteFullBeat: Boolean
+  def inner_coh: ManagerMetadata
+  def trackerId: Int
+
+  // Secondary miss queue holds transaction metadata used to make grants
+  lazy val ignt_q = Module(new Queue(
+        new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })),
+        1 + nSecondaryMisses))
+
+  val pending_ignt = Wire(Bool())
+  val ignt_data_idx = Wire(UInt())
+  val ignt_data_done = Wire(Bool())
+  val ifin_counter = Wire(new TwoWayBeatCounterStatus)
+  val pending_put_data = Reg(init=Bits(0, width = innerDataBeats))
+  val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats))
+
+  def iacq_same_xact: Bool = {
+    (xact_iacq.client_xact_id === io.iacq().client_xact_id) &&
+      xact_iacq.hasMultibeatData() &&
+      pending_ignt &&
+      pending_put_data(io.iacq().addr_beat)
+  }
+  def iacq_can_merge: Bool
+  def iacq_is_allocating: Bool = state === s_idle && io.alloc.iacq.should && io.inner.acquire.valid
+  def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid
+
+  def innerAcquire(can_alloc: Bool, next: UInt) {
+
+    // Enqueue some metadata information that we'll use to make coherence updates with later
+    ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && io.iacq().first())
+    ignt_q.io.enq.bits := io.iacq()
+
+    // Use the outputs of the queue to make further messages
+    xact_iacq := Mux(ignt_q.io.deq.valid, ignt_q.io.deq.bits, ignt_q.io.enq.bits)
+    xact_addr_beat := xact_iacq.addr_beat
+    pending_ignt := ignt_q.io.count > UInt(0)
+
+    // Track whether any beats are missing from a PutBlock
+    when (state =/= s_idle || io.alloc.iacq.should) {
+      pending_put_data := (pending_put_data &
+          dropPendingBitWhenBeatHasData(io.inner.acquire)) |
+          addPendingBitsOnFirstBeat(io.inner.acquire)
+    }
+
+    // Intialize transaction metadata for accepted Acquire
+    when(iacq_is_allocating) {
+      xact_addr_block := io.iacq().addr_block
+      xact_allocate := io.iacq().allocate() && can_alloc
+      xact_amo_shift_bytes := io.iacq().amo_shift_bytes()
+      xact_op_code := io.iacq().op_code()
+      xact_addr_byte := io.iacq().addr_byte()
+      xact_op_size := io.iacq().op_size()
+      // Make sure to collect all data from a PutBlock
+      pending_put_data := Mux(
+        io.iacq().isBuiltInType(Acquire.putBlockType),
+        dropPendingBitWhenBeatHasData(io.inner.acquire),
+        UInt(0))
+      pending_ignt_data := UInt(0)
+      state := next
+    }
+
+    scoreboard += (pending_put_data.orR)
+  }
+
+  def innerGrant(
+      data: UInt = io.ognt().data,
+      external_pending: Bool = Bool(false),
+      add: UInt = UInt(0)) {
+    // Track the number of outstanding inner.finishes
+    connectTwoWayBeatCounters(
+      status = ifin_counter,
+      up = io.inner.grant,
+      down = io.inner.finish,
+      max = nSecondaryMisses,
+      trackUp = (g: Grant) => g.requiresAck())
+
+    // Track which beats are ready for response
+    when(!iacq_is_allocating) {
+      pending_ignt_data := (pending_ignt_data & dropPendingBitWhenBeatHasData(io.inner.grant)) |
+                           addPendingBitWhenBeatHasData(io.inner.release) |
+                           addPendingBitWhenBeatHasData(io.outer.grant) |
+                           add
+    }
+
+    // We can issue a grant for a pending write once all data is
+    // received and committed to the data array or outer memory
+    val ignt_ack_ready = !(state === s_idle ||
+                           state === s_meta_read ||
+                           pending_put_data.orR)
+
+    val ignt_from_iacq = inner_coh.makeGrant(
+                            sec = ignt_q.io.deq.bits,
+                            manager_xact_id = UInt(trackerId), 
+                            data = data)
+
+    // Make the Grant message using the data stored in the secondary miss queue
+    val (cnt, done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat)
+    ignt_data_idx := cnt
+    ignt_data_done := done
+    ignt_q.io.deq.ready := Bool(false)
+    when(!vol_ignt_counter.pending) {
+      ignt_q.io.deq.ready := ignt_data_done
+      io.inner.grant.bits := ignt_from_iacq
+      io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter
+      when (state === s_busy && pending_ignt) {
+        io.inner.grant.valid := !external_pending && 
+                                  Mux(io.ignt().hasData(), pending_ignt_data(ignt_data_idx), ignt_ack_ready)
+      }
+    }
+
+    // We must wait for as many Finishes as we sent Grants
+    io.inner.finish.ready := state === s_busy
+
+    scoreboard += (pending_ignt, ifin_counter.pending)
+  }
+
+}
+
+trait EmitsOuterAcquires extends AcceptsInnerAcquires {
+  val ognt_counter = Wire(new TwoWayBeatCounterStatus)
+
+  // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory:
+  //
+  // If we're allocating in this cache, we can use the current metadata
+  // to make an appropriate custom Acquire, otherwise we copy over the
+  // built-in Acquire from the inner TL to the outer TL
+  def outerAcquire(
+      caching: Bool,
+      coh: ClientMetadata,
+      buffering: Bool = Bool(true),
+      data: UInt = io.iacq().data,
+      wmask: UInt = io.iacq().wmask(),
+      next: UInt = s_busy) {
+
+    // Tracks outstanding Acquires, waiting for their matching Grant.
+    connectTwoWayBeatCounters(
+      status = ognt_counter,
+      up = io.outer.acquire,
+      down = io.outer.grant,
+      beat = xact_addr_beat,
+      trackDown = (g: Grant) => !g.isVoluntary())
+
+    io.outer.acquire.valid := state === s_outer_acquire &&
+                                (xact_allocate || 
+                                  Mux(buffering,
+                                      !pending_put_data(ognt_counter.up.idx),
+                                      io.inner.acquire.valid))
+
+    io.outer.acquire.bits :=
+      Mux(caching,
+        coh.makeAcquire(
+          op_code = xact_op_code,
+          client_xact_id = UInt(0),
+          addr_block = xact_addr_block),
+        BuiltInAcquireBuilder(
+          a_type = xact_iacq.a_type,
+          client_xact_id = UInt(0),
+          addr_block = xact_addr_block,
+          addr_beat = ognt_counter.up.idx,
+          data = data,
+          addr_byte = xact_addr_byte,
+          operand_size = xact_op_size,
+          opcode = xact_op_code,
+          wmask = wmask,
+          alloc = Bool(false))
+          (p.alterPartial({ case TLId => p(OuterTLId)})))
+
+    when(state === s_outer_acquire && ognt_counter.up.done) { state := next }
+
+    io.outer.grant.ready := state === s_busy
+
+    scoreboard += ognt_counter.pending
+  }
+}
+
+abstract class VoluntaryReleaseTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
+    with AcceptsVoluntaryReleases
+    with RoutesInParent {
+  def irel_can_merge = Bool(false)
+  def irel_same_xact = io.irel().conflicts(xact_addr_block) &&
+                          io.irel().isVoluntary() &&
+                          pending_irel_data.orR
+}
+
+abstract class AcquireTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p)
+    with AcceptsInnerAcquires
+    with EmitsOuterAcquires
+    with EmitsInnerProbes
+    with RoutesInParent {
+}