Merge branch 'master' into new-llc

Conflicts:
	src/main/scala/coherence.scala
	src/main/scala/memserdes.scala
	src/main/scala/tilelink.scala

uncore/src/main/scala/cache.scala

@@ -99,6 +99,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule {
 
 abstract trait L2HellaCacheParameters extends CacheParameters 
   with CoherenceAgentParameters
+  with TileLinkParameters
 
 abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters
 abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters
@@ -176,16 +177,16 @@ class L2MetadataArray extends L2HellaCacheModule {
 
 class L2DataReadReq extends L2HellaCacheBundle with HasL2Id {
   val way_en = Bits(width = nWays)
-  val addr   = Bits(width = params(TLAddrBits))
+  val addr   = Bits(width = tlAddrBits)
 }
 
 class L2DataWriteReq extends L2DataReadReq {
-  val wmask  = Bits(width = params(TLWriteMaskBits))
+  val wmask  = Bits(width = tlWriteMaskBits)
-  val data   = Bits(width = params(TLDataBits))
+  val data   = Bits(width = tlDataBits)
 }
 
-class L2DataResp extends Bundle with HasL2Id {
+class L2DataResp extends Bundle with HasL2Id with TileLinkParameters {
-  val data   = Bits(width = params(TLDataBits))
+  val data   = Bits(width = tlDataBits)
 }
 
 class L2DataArray extends L2HellaCacheModule {
@@ -295,7 +296,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac
   val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_)
   val block_releases = Bool(false)
   val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b}
-  val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id)
+  val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), conflict_idx)
   for( i <- 0 until trackerList.size ) {
     val t = trackerList(i).io.inner
     t.release.bits := release.bits 
@@ -380,7 +381,8 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
 
   io.inner.grant.bits.header.src := UInt(bankId)
   io.inner.grant.bits.header.dst := init_client_id
-  io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh),
+  io.inner.grant.bits.payload := Grant(Bool(false),
+                                        co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh),
                                         xact.client_xact_id,
                                         UInt(trackerId))
 
@@ -436,7 +438,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou
     is(s_grant) {
       io.inner.grant.valid := Bool(true)
       when(io.inner.grant.ready) { 
-        state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type),
+        state := Mux(co.requiresAckForGrant(c_gnt.payload),
           s_busy, s_idle) 
       }
     }
@@ -463,11 +465,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
   val pending_probes = Reg(init = co.dir().flush)
   val curr_p_id = co.dir().next(pending_probes)
   
-  val is_uncached = co.messageIsUncached(xact)
+  val is_uncached = xact.uncached
   val tag_match = xact_internal.tag_match
   val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh)
   val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh)
-  val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh)
+  val needs_probes = co.requiresProbes(xact, xact_internal.meta.coh)
   //TODO: does allocate
 
   io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && 
@@ -485,28 +487,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
                                     c_gnt.header.dst)
 
   val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits))
-  val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, 
+  val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), 
-                                        xact.addr, 
+                          { case TLId => outerId })
-                                        UInt(trackerId), 
+  val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer),
-                                        xact.data), { case TLId => outerId })
+                          { case TLId => outerId })
-  val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, 
+  val outer_read = Bundle(UncachedRead( xact.addr, UInt(trackerId)), { case TLId => outerId })
-                                        addr_wb,
+
-                                        UInt(trackerId), 
-                                        wb_buffer), { case TLId => outerId })
-  val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, 
-                                        xact.addr, 
-                                        UInt(trackerId)), { case TLId => outerId })
   io.outer.acquire.valid := Bool(false)
   io.outer.acquire.bits.payload := outer_read //default
   io.outer.acquire.bits.header.src := UInt(bankId)
   io.outer.grant.ready := Bool(true) //grant.data -> xact.data
 
-  val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh),
+  val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr)
-                                xact.addr,
+  val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, addr_wb)
-                                UInt(trackerId))
-  val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback,
-                                addr_wb,
-                                UInt(trackerId))
   //TODO inner_probe_mprb
   io.inner.probe.valid := Bool(false)
   io.inner.probe.bits.header.src := UInt(bankId)
@@ -515,11 +508,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
                                       cprb_for_mwb,
                                       cprb_for_cacq)
 
-  val cgnt_for_cacq = Grant(co.getGrantType(xact, xact_internal.meta.coh),
+  val cgnt_for_cacq = Grant(xact.uncached, co.getGrantType(xact, xact_internal.meta.coh),
                               xact.client_xact_id,
                               UInt(trackerId),
                               xact.data)
-  val cgnt_for_cwb = Grant(crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0))
+  val cgnt_for_cwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0))
   io.inner.grant.valid := Bool(false)
   io.inner.grant.bits.header.src := UInt(bankId)
   io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id)
@@ -569,12 +562,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
         val _tag_match = io.meta_resp.bits.tag_match
         val _needs_writeback = !_tag_match && co.needsWriteback(coh)
         val _is_hit = _tag_match && co.isHit(xact, coh)
-        val _needs_probes = co.requiresProbes(xact.a_type, coh)
+        val _needs_probes = co.requiresProbes(xact, coh)
         xact_internal := io.meta_resp.bits
         when(_needs_probes) {
           val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent
           val mask_self = mask_incoherent & 
-                            ~(!(co.requiresSelfProbe(xact.a_type) || _needs_writeback) << init_client_id)
+                            ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << init_client_id)
           pending_probes := mask_self
           release_count := co.dir().count(mask_self)
           crel_had_data := Bool(false)
@@ -610,7 +603,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
         when(co.isVoluntary(c_rel.payload)) {
           crel_was_voluntary := Bool(true)
           crel_wb_src := c_rel.header.src
-          crel_wb_g_type := co.getGrantType(c_rel.payload, xact_internal.meta.coh)
+          crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh)
         }
         when(!co.isVoluntary(c_rel.payload)) {
           release_count := release_count - UInt(1)
@@ -692,7 +685,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St
     is(s_grant) {
       io.inner.grant.valid := Bool(true)
       when(io.inner.grant.ready) { 
-        state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type),
+        state := Mux(co.requiresAckForGrant(c_gnt.payload),
           s_busy, s_idle) 
       }
     }

uncore/src/main/scala/coherence.scala

@@ -61,15 +61,15 @@ abstract class DirectoryRepresentation(val width: Int) {
   def full(s: UInt): UInt
 }
 
-class NullRepresentation extends DirectoryRepresentation(1) {
+class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) {
   def pop(prev: UInt, id: UInt) = UInt(0)
   def push(prev: UInt, id: UInt) = UInt(0)
   def flush  = UInt(0)
   def none(s: UInt) = Bool(false)
   def one(s: UInt) = Bool(false)
-  def count(s: UInt) = UInt(0)
+  def count(s: UInt) = UInt(nClients)
   def next(s: UInt) = UInt(0)
-  def full(s: UInt) = UInt(0)
+  def full(s: UInt) = SInt(-1, width = nClients).toUInt
 }
 
 class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) {
@@ -124,58 +124,49 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) {
   def getReleaseTypeOnVoluntaryWriteback(): UInt
   def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt
   def getGrantType(a: Acquire, m: MasterMetadata): UInt
-  def getGrantType(r: Release, m: MasterMetadata): UInt
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt
 
-  def messageHasData (rel: SourcedMessage): Bool
+  def messageHasData(rel: SourcedMessage): Bool
-  def messageUpdatesDataArray (reply: Grant): Bool
+  def messageUpdatesDataArray(g: Grant): Bool
-  def messageIsUncached(acq: Acquire): Bool
   def isVoluntary(rel: Release): Bool
   def isVoluntary(gnt: Grant): Bool
-
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool
-  def requiresOuterRead(a_type: UInt): Bool
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool
-  def requiresOuterWrite(a_type: UInt): Bool
+  def requiresSelfProbe(a: Acquire): Bool
-  def requiresSelfProbe(a_type: UInt): Bool
+  def requiresProbes(a: Acquire, m: MasterMetadata): Bool
-  def requiresProbes(a_type: UInt, m: MasterMetadata): Bool
+  def requiresAckForGrant(g: Grant): Bool
-  def requiresAckForGrant(g_type: UInt): Bool
+  def requiresAckForRelease(r: Release): Bool
-  def requiresAckForRelease(r_type: UInt): Bool
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool
+
+  def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = {
+    MuxLookup(a.a_type, Grant.uncachedRead, Array(
+      Acquire.uncachedRead -> Grant.uncachedRead,
+      Acquire.uncachedWrite -> Grant.uncachedWrite,
+      Acquire.uncachedAtomic -> Grant.uncachedAtomic
+    ))
+  }
 }
 
-trait UncachedTransactions {
+class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) {
-  def getUncachedReadAcquireType: UInt
-  def getUncachedWriteAcquireType: UInt
-  def getUncachedReadWordAcquireType: UInt
-  def getUncachedWriteWordAcquireType: UInt
-  def getUncachedAtomicAcquireType: UInt
-  def isUncachedReadTransaction(acq: Acquire): Bool
-  def getUncachedReadGrantType: UInt
-}
-
-abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir)
-  with UncachedTransactions
-
-class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) {
   def nClientStates = 2
   def nMasterStates = 2
-  def nAcquireTypes = 6
+  def nAcquireTypes = 1
   def nProbeTypes = 2
   def nReleaseTypes = 5
-  def nGrantTypes = 7
+  def nGrantTypes = 2
 
   val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates)
   val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates)
 
-  val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes)
+  val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes)
   val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes)
   val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes)
-  val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes)
+  val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes)
 
-  val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) 
-  val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
   val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData)
-  val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached)
+  val hasDataGrantTypeVec = Vec(grantReadExclusive)
 
   def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m)
   def isValid (m: ClientMetadata): Bool = m.state != clientInvalid
@@ -201,15 +192,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit
       M_CLN -> clientValid
     )))(this)
   def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV)
-  def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(
+  def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = 
-    MuxLookup(incoming.g_type, clientInvalid, Array(
+    ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this)
-      grantReadExclusive -> clientValid,
-      grantReadUncached  -> clientInvalid,
-      grantWriteUncached -> clientInvalid,
-      grantReadWordUncached -> clientInvalid,
-      grantWriteWordUncached -> clientInvalid,
-      grantAtomicUncached -> clientInvalid
-    )))(this)
   def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata(
     MuxLookup(incoming.p_type, m.state, Array(
       probeInvalidate -> clientInvalid,
@@ -232,16 +216,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit
     ))
   }
 
-  def getUncachedReadAcquireType = acquireReadUncached
-  def getUncachedWriteAcquireType = acquireWriteUncached
-  def getUncachedReadWordAcquireType = acquireReadWordUncached
-  def getUncachedWriteWordAcquireType = acquireWriteWordUncached
-  def getUncachedAtomicAcquireType = acquireAtomicUncached
-  def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
-  def getUncachedReadGrantType = grantReadUncached
-
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
-  def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck
+  def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck
 
   def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive
   def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive
@@ -260,80 +236,63 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit
   }
 
   def messageHasData(msg: SourcedMessage) = msg match {
-    case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type)
+    case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false))
-    case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) 
+    case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type))
     case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) 
     case _ => Bool(false)
   }
-  def messageUpdatesDataArray (reply: Grant): Bool = {
+  def messageUpdatesDataArray(g: Grant): Bool = {
-    (reply.g_type === grantReadExclusive)
+    Mux(g.uncached, Bool(false), 
+      (g.g_type === grantReadExclusive))
   }
-  def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type)
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2)
 
-  def getGrantType(a: Acquire, m: MasterMetadata): UInt = {
+  def getGrantType(a: Acquire, m: MasterMetadata): UInt = 
-    MuxLookup(a.a_type, grantReadUncached, Array(
+    Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive)
-      acquireReadExclusive -> grantReadExclusive,
-      acquireReadUncached  -> grantReadUncached,
-      acquireWriteUncached -> grantWriteUncached,
-      acquireReadWordUncached  -> grantReadWordUncached,
-      acquireWriteWordUncached -> grantWriteWordUncached,
-      acquireAtomicUncached -> grantAtomicUncached
-    ))
-  }
 
-  def getGrantType(r: Release, m: MasterMetadata): UInt = {
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck
-    MuxLookup(r.r_type, grantReadUncached, Array(
-      releaseVoluntaryInvalidateData -> grantVoluntaryAck
-    ))
-  }
 
   def getProbeType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, probeCopy, Array(
+    Mux(a.uncached, 
-      acquireReadExclusive -> probeInvalidate, 
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireReadUncached -> probeCopy, 
+        Acquire.uncachedRead -> probeCopy, 
-      acquireWriteUncached -> probeInvalidate,
+        Acquire.uncachedWrite -> probeInvalidate,
-      acquireReadWordUncached -> probeCopy, 
+        Acquire.uncachedAtomic -> probeInvalidate
-      acquireWriteWordUncached -> probeInvalidate,
+      )), probeInvalidate)
-      acquireAtomicUncached -> probeInvalidate
-    ))
   }
-  def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
 
-  def requiresOuterRead(a_type: UInt) = {
+  def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
-      (a_type != acquireWriteUncached)
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata) = 
-  }
+    Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true))
-  def requiresOuterWrite(a_type: UInt) = {
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata) =
-      (a_type === acquireWriteUncached)
+    Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false))
-  }
+
-  def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck
+  def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck
-  def requiresAckForRelease(r_type: UInt) = Bool(false)
+  def requiresAckForRelease(r: Release) = Bool(false)
-  def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached
+  def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead
-  def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers)
+  def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers)
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData)
 }
 
-class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) {
+class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) {
   def nClientStates = 3
   def nMasterStates = 2
-  def nAcquireTypes = 7
+  def nAcquireTypes = 2
   def nProbeTypes = 3
   def nReleaseTypes = 7
-  def nGrantTypes = 8
+  def nGrantTypes = 2
 
   val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates)
   val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates)
 
-  val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes)
+  val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes)
   val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes)
   val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes)
-  val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes)
+  val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes)
 
-  val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) 
-  val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) 
   val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData)
-  val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached)
+  val hasDataGrantTypeVec = Vec(grantReadExclusive)
 
   def isHit (cmd: UInt, m: ClientMetadata) = isValid(m)
   def isValid (m: ClientMetadata) = m.state != clientInvalid
@@ -341,7 +300,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
   def isValid (m: MasterMetadata) = m.state != masterInvalid
 
   def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = {
-    (isRead(cmd) && messageIsUncached(outstanding)) ||
+    (isRead(cmd) && outstanding.uncached) ||
       (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty))
   }
   def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = {
@@ -365,16 +324,9 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
     )))(this)
   def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV)
   def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(
-    MuxLookup(incoming.g_type, clientInvalid, Array(
+    Mux(incoming.uncached, clientInvalid,
-      grantReadExclusive  -> Mux(outstanding.a_type === acquireReadExclusiveDirty, 
+      Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, 
-        clientExclusiveDirty, clientExclusiveClean),
+        clientExclusiveClean)))(this)
-      grantReadExclusiveAck -> clientExclusiveDirty, 
-      grantReadUncached -> clientInvalid,
-      grantWriteUncached -> clientInvalid,
-      grantReadWordUncached -> clientInvalid,
-      grantWriteWordUncached -> clientInvalid,
-      grantAtomicUncached -> clientInvalid
-    )))(this)
   def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata(
     MuxLookup(incoming.p_type, m.state, Array(
       probeInvalidate -> clientInvalid,
@@ -394,21 +346,12 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
     val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this)
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
     MuxBundle(uncached, Array(
-      g.is(grantReadExclusive) -> cached,
+      g.is(grantReadExclusive) -> cached
-      g.is(grantReadExclusiveAck) -> cached
     ))
   }
 
-  def getUncachedReadAcquireType = acquireReadUncached
-  def getUncachedWriteAcquireType = acquireWriteUncached
-  def getUncachedReadWordAcquireType = acquireReadWordUncached
-  def getUncachedWriteWordAcquireType = acquireWriteWordUncached
-  def getUncachedAtomicAcquireType = acquireAtomicUncached
-  def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
-  def getUncachedReadGrantType = grantReadUncached
-
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
-  def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck
+  def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck
 
   def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = {
     Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean)
@@ -433,84 +376,65 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
   }
 
   def messageHasData(msg: SourcedMessage) = msg match {
-    case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type)
+    case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false))
-    case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) 
+    case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type))
     case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) 
     case _ => Bool(false)
   }
-  def messageUpdatesDataArray (reply: Grant): Bool = {
+  def messageUpdatesDataArray(g: Grant): Bool = {
-    (reply.g_type === grantReadExclusive)
+    Mux(g.uncached, Bool(false), 
+      (g.g_type === grantReadExclusive))
   }
-  def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type)
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2)
 
   def getGrantType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, grantReadUncached, Array(
+    Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive)
-      acquireReadExclusiveClean -> grantReadExclusive,
-      acquireReadExclusiveDirty -> grantReadExclusive,
-      acquireReadUncached  -> grantReadUncached,
-      acquireWriteUncached -> grantWriteUncached,
-      acquireReadWordUncached  -> grantReadWordUncached,
-      acquireWriteWordUncached -> grantWriteWordUncached,
-      acquireAtomicUncached -> grantAtomicUncached
-    ))
   }
-  def getGrantType(r: Release, m: MasterMetadata): UInt = {
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck
-    MuxLookup(r.r_type, grantReadUncached, Array(
-      releaseVoluntaryInvalidateData -> grantVoluntaryAck
-    ))
-  }
-
 
   def getProbeType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, probeCopy, Array(
+    Mux(a.uncached, 
-      acquireReadExclusiveClean -> probeInvalidate,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireReadExclusiveDirty -> probeInvalidate, 
+        Acquire.uncachedRead -> probeCopy, 
-      acquireReadUncached -> probeCopy, 
+        Acquire.uncachedWrite -> probeInvalidate,
-      acquireWriteUncached -> probeInvalidate,
+        Acquire.uncachedAtomic -> probeInvalidate
-      acquireReadWordUncached -> probeCopy, 
+      )), probeInvalidate)
-      acquireWriteWordUncached -> probeInvalidate,
-      acquireAtomicUncached -> probeInvalidate
-    ))
   }
   def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
 
-  def requiresOuterRead(a_type: UInt) = {
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata) = 
-      (a_type != acquireWriteUncached)
+    Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true))
-  }
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata) =
-  def requiresOuterWrite(a_type: UInt) = {
+    Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false))
-      (a_type === acquireWriteUncached)
+
-  }
+  def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck
-  def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck
+  def requiresAckForRelease(r: Release) = Bool(false)
-  def requiresAckForRelease(r_type: UInt) = Bool(false)
+  def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead
-  def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached
+  def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers)
-  def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers)
 
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData)
 }
 
-class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) {
+class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) {
   def nClientStates = 3
   def nMasterStates = 2
-  def nAcquireTypes = 7
+  def nAcquireTypes = 2
   def nProbeTypes = 3
   def nReleaseTypes = 7
-  def nGrantTypes = 9
+  def nGrantTypes = 3
 
   val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates)
   //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates)
   val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates)
 
-  val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes)
+  val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes)
   val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes)
   val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes)
-  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes)
+  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes)
 
-  val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) 
-  val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
   val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData)
-  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached)
+  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive)
 
   def isHit (cmd: UInt, m: ClientMetadata): Bool = {
     Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty),
@@ -523,7 +447,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
   def isValid (m: MasterMetadata) = m.state != masterInvalid
 
   def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = {
-    (isRead(cmd) && messageIsUncached(outstanding)) || 
+    (isRead(cmd) && outstanding.uncached) || 
       (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive))
   }
   def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = {
@@ -546,16 +470,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
     )))(this)
   def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV)
   def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(
-    MuxLookup(incoming.g_type, clientInvalid, Array(
+    Mux(incoming.uncached, clientInvalid,
-      grantReadShared -> clientShared,
+      Mux(incoming.g_type === grantReadShared, clientShared, 
-      grantReadExclusive  -> clientExclusiveDirty,
+        clientExclusiveDirty)))(this)
-      grantReadExclusiveAck -> clientExclusiveDirty, 
-      grantReadUncached -> clientInvalid,
-      grantWriteUncached -> clientInvalid,
-      grantReadWordUncached -> clientInvalid,
-      grantWriteWordUncached -> clientInvalid,
-      grantAtomicUncached -> clientInvalid
-    )))(this)
   def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata(
     MuxLookup(incoming.p_type, m.state, Array(
       probeInvalidate -> clientInvalid,
@@ -576,21 +493,12 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
     val uncached = MasterMetadata(masterValid, m.sharers)(this)
     MuxBundle(uncached, Array(
       g.is(grantReadShared) -> cached,
-      g.is(grantReadExclusive) -> cached,
+      g.is(grantReadExclusive) -> cached
-      g.is(grantReadExclusiveAck) -> cached
     ))
   }
 
-  def getUncachedReadAcquireType = acquireReadUncached
-  def getUncachedWriteAcquireType = acquireWriteUncached
-  def getUncachedReadWordAcquireType = acquireReadWordUncached
-  def getUncachedWriteWordAcquireType = acquireWriteWordUncached
-  def getUncachedAtomicAcquireType = acquireAtomicUncached
-  def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
-  def getUncachedReadGrantType = grantReadUncached
-
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
-  def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck
+  def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck
 
   def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = {
     Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared)
@@ -615,80 +523,72 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi
   }
 
   def messageHasData(msg: SourcedMessage) = msg match {
-    case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type)
+    case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false))
-    case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) 
+    case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type))
     case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) 
     case _ => Bool(false)
   }
-  def messageUpdatesDataArray (reply: Grant): Bool = {
+  def messageUpdatesDataArray(g: Grant): Bool = {
-    (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive)
+    Mux(g.uncached, Bool(false), 
+      (g.g_type === grantReadShared || g.g_type === grantReadExclusive))
   }
-  def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type)
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2)
 
   def getGrantType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, grantReadUncached, Array(
+    Mux(a.uncached, getGrantTypeForUncached(a, m),
-      acquireReadShared    -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive),
+      Mux(a.a_type === acquireReadShared,
-      acquireReadExclusive -> grantReadExclusive,
+        Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive),
-      acquireReadUncached  -> grantReadUncached,
+        grantReadExclusive))
-      acquireWriteUncached -> grantWriteUncached,
-      acquireReadWordUncached  -> grantReadWordUncached,
-      acquireWriteWordUncached -> grantWriteWordUncached,
-      acquireAtomicUncached -> grantAtomicUncached
-    ))
-  }
-  def getGrantType(r: Release, m: MasterMetadata): UInt = {
-    MuxLookup(r.r_type, grantReadUncached, Array(
-      releaseVoluntaryInvalidateData -> grantVoluntaryAck
-    ))
   }
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck
 
   def getProbeType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, probeCopy, Array(
+    Mux(a.uncached, 
-      acquireReadShared -> probeDowngrade,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireReadExclusive -> probeInvalidate, 
+        Acquire.uncachedRead -> probeCopy, 
-      acquireReadUncached -> probeCopy, 
+        Acquire.uncachedWrite -> probeInvalidate,
-      acquireWriteUncached -> probeInvalidate
+        Acquire.uncachedAtomic -> probeInvalidate
-    ))
+      )),
+      MuxLookup(a.a_type, probeInvalidate, Array(
+        acquireReadShared -> probeDowngrade,
+        acquireReadExclusive -> probeInvalidate
+      )))
   }
   def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
 
-  def requiresOuterRead(a_type: UInt) = {
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata) = 
-      (a_type != acquireWriteUncached)
+    Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true))
-  }
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata) =
-  def requiresOuterWrite(a_type: UInt) = {
+    Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false))
-      (a_type === acquireWriteUncached)
+
-  }
+  def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck
-  def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck
+  def requiresAckForRelease(r: Release) = Bool(false)
-  def requiresAckForRelease(r_type: UInt) = Bool(false)
+  def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead
-  def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached
+  def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers)
-  def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers)
 
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData)
 }
 
-class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) {
+class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) {
   def nClientStates = 4
   def nMasterStates = 2
-  def nAcquireTypes = 7
+  def nAcquireTypes = 2
   def nProbeTypes = 3
   def nReleaseTypes = 7
-  def nGrantTypes = 9
+  def nGrantTypes = 4
 
   val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates)
   //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates)
   val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates)
 
-  val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes)
+  val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes)
   val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes)
   val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes)
-  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes)
+  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes)
 
-  val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) 
-  val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) 
   val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData)
-  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached)
+  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive)
 
   def isHit (cmd: UInt, m: ClientMetadata): Bool = {
     Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty),
@@ -701,7 +601,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW
   def isValid (m: MasterMetadata) = m.state != masterInvalid
 
   def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = {
-    (isRead(cmd) && messageIsUncached(outstanding)) ||
+    (isRead(cmd) && outstanding.uncached) ||
       (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive))
   }
   def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = {
@@ -725,16 +625,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW
     )))(this)
   def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV)
   def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(
-    MuxLookup(incoming.g_type, clientInvalid, Array(
+    Mux(incoming.uncached, clientInvalid,
-      grantReadShared -> clientShared,
+      MuxLookup(incoming.g_type, clientInvalid, Array(
-      grantReadExclusive  -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean),
+        grantReadShared -> clientShared,
-      grantReadExclusiveAck -> clientExclusiveDirty, 
+        grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, 
-      grantReadUncached -> clientInvalid,
+                                clientExclusiveDirty, clientExclusiveClean),
-      grantWriteUncached -> clientInvalid,
+        grantReadExclusiveAck -> clientExclusiveDirty
-      grantReadWordUncached -> clientInvalid,
+      ))))(this)
-      grantWriteWordUncached -> clientInvalid,
-      grantAtomicUncached -> clientInvalid
-    )))(this)
   def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata(
     MuxLookup(incoming.p_type, m.state, Array(
       probeInvalidate -> clientInvalid,
@@ -760,16 +657,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW
     ))
   }
 
-  def getUncachedReadAcquireType = acquireReadUncached
-  def getUncachedWriteAcquireType = acquireWriteUncached
-  def getUncachedReadWordAcquireType = acquireReadWordUncached
-  def getUncachedWriteWordAcquireType = acquireWriteWordUncached
-  def getUncachedAtomicAcquireType = acquireAtomicUncached
-  def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
-  def getUncachedReadGrantType = grantReadUncached
-
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
-  def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck
+  def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck
 
   def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = {
     Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared)
@@ -794,84 +683,71 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW
   }
 
   def messageHasData(msg: SourcedMessage) = msg match {
-    case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type)
+    case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false))
-    case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) 
+    case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type))
     case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) 
     case _ => Bool(false)
   }
-  def messageUpdatesDataArray (reply: Grant): Bool = {
+  def messageUpdatesDataArray(g: Grant): Bool = {
-    (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive)
+    Mux(g.uncached, Bool(false), 
+      (g.g_type === grantReadShared || g.g_type === grantReadExclusive))
   }
-  def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type)
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2)
 
   def getGrantType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, grantReadUncached, Array(
+    Mux(a.uncached, getGrantTypeForUncached(a, m),
-      acquireReadShared    -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive),
+      Mux(a.a_type === acquireReadShared,
-      acquireReadExclusive -> grantReadExclusive,
+        Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive),
-      acquireReadUncached  -> grantReadUncached,
+        grantReadExclusive))
-      acquireWriteUncached -> grantWriteUncached,
-      acquireReadWordUncached  -> grantReadWordUncached,
-      acquireWriteWordUncached -> grantWriteWordUncached,
-      acquireAtomicUncached -> grantAtomicUncached
-    ))
   }
-  def getGrantType(r: Release, m: MasterMetadata): UInt = {
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck
-    MuxLookup(r.r_type, grantReadUncached, Array(
-      releaseVoluntaryInvalidateData -> grantVoluntaryAck
-    ))
-  }
-
 
   def getProbeType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, probeCopy, Array(
+    Mux(a.uncached, 
-      acquireReadShared -> probeDowngrade,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireReadExclusive -> probeInvalidate, 
+        Acquire.uncachedRead -> probeCopy, 
-      acquireReadUncached -> probeCopy, 
+        Acquire.uncachedWrite -> probeInvalidate,
-      acquireWriteUncached -> probeInvalidate,
+        Acquire.uncachedAtomic -> probeInvalidate
-      acquireReadWordUncached -> probeCopy, 
+      )),
-      acquireWriteWordUncached -> probeInvalidate,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireAtomicUncached -> probeInvalidate
+        acquireReadShared -> probeDowngrade,
-    ))
+        acquireReadExclusive -> probeInvalidate
+      )))
   }
   def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
 
-  def requiresOuterRead(a_type: UInt) = {
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata) = 
-      (a_type != acquireWriteUncached)
+    Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true))
-  }
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata) =
-  def requiresOuterWrite(a_type: UInt) = {
+    Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false))
-      (a_type === acquireWriteUncached)
-  }
 
-  def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck
+  def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck
-  def requiresAckForRelease(r_type: UInt) = Bool(false)
+  def requiresAckForRelease(r: Release) = Bool(false)
-  def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached
+  def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead
-  def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers)
+  def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers)
 
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData)
 }
 
-class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) {
+class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) {
   def nClientStates = 7
   def nMasterStates = 2
-  def nAcquireTypes = 8
+  def nAcquireTypes = 3
   def nProbeTypes = 4
   def nReleaseTypes = 11
-  def nGrantTypes = 9
+  def nGrantTypes = 5
 
   val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates)
   //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates)
   val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates)
 
-  val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes)
+  val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes)
   val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes)
   val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes)
-  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes)
+  val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes)
 
-  val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) 
+  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory)
-  val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
-  val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached)
   val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory)
 
   def isHit (cmd: UInt, m: ClientMetadata): Bool = {
@@ -884,7 +760,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo
   def isValid (m: MasterMetadata) = m.state != masterInvalid
 
   def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = {
-    (isRead(cmd) && messageIsUncached(outstanding)) ||
+    (isRead(cmd) && outstanding.uncached) ||
       (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers))
   }
   def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = {
@@ -909,22 +785,18 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo
     )))(this)
   def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV)
   def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(
-    MuxLookup(incoming.g_type, clientInvalid, Array(
+    Mux(incoming.uncached, clientInvalid,
-      grantReadShared -> clientShared,
+      MuxLookup(incoming.g_type, clientInvalid, Array(
-      grantReadExclusive  -> MuxLookup(outstanding.a_type, clientExclusiveDirty,  Array(
+        grantReadShared -> clientShared,
-                                   acquireReadExclusive -> clientExclusiveDirty,
+        grantReadExclusive  -> MuxLookup(outstanding.a_type, clientExclusiveDirty,  Array(
-                                   acquireReadShared -> clientExclusiveClean)),
+                                     acquireReadExclusive -> clientExclusiveDirty,
-      grantReadExclusiveAck -> clientExclusiveDirty, 
+                                     acquireReadShared -> clientExclusiveClean)),
-      grantReadUncached -> clientInvalid,
+        grantReadExclusiveAck -> clientExclusiveDirty, 
-      grantWriteUncached -> clientInvalid,
+        grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array(
-      grantReadWordUncached -> clientInvalid,
+                                    acquireInvalidateOthers -> clientMigratoryDirty,
-      grantWriteWordUncached -> clientInvalid,
+                                    acquireReadExclusive -> clientMigratoryDirty,
-      grantAtomicUncached -> clientInvalid,
+                                    acquireReadShared -> clientMigratoryClean))
-      grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array(
+      ))))(this)
-                                  acquireInvalidateOthers -> clientMigratoryDirty,
-                                  acquireReadExclusive -> clientMigratoryDirty,
-                                  acquireReadShared -> clientMigratoryClean))
-    )))(this)
   def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata(
     MuxLookup(incoming.p_type, m.state, Array(
       probeInvalidate -> clientInvalid,
@@ -960,16 +832,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo
   }
 
 
-  def getUncachedReadAcquireType = acquireReadUncached
-  def getUncachedWriteAcquireType = acquireWriteUncached
-  def getUncachedReadWordAcquireType = acquireReadWordUncached
-  def getUncachedWriteWordAcquireType = acquireWriteWordUncached
-  def getUncachedAtomicAcquireType = acquireAtomicUncached
-  def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
-  def getUncachedReadGrantType = grantReadUncached
-
   def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
-  def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck
+  def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck
 
   def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = {
     Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared)
@@ -996,60 +860,52 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo
   }
 
   def messageHasData(msg: SourcedMessage) = msg match {
-    case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type)
+    case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false))
-    case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) 
+    case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type))
     case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) 
     case _ => Bool(false)
   }
-  def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type)
+  def messageUpdatesDataArray(g: Grant): Bool = {
-  def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type)
+    Mux(g.uncached, Bool(false), 
+      Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type))
+  }
 
   def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2)
 
   def getGrantType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, grantReadUncached, Array(
+    Mux(a.uncached, getGrantTypeForUncached(a, m),
-      acquireReadShared    -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type???
+      MuxLookup(a.a_type, grantReadShared, Array(
-      acquireReadExclusive -> grantReadExclusive,                                            
+        acquireReadShared    -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive),
-      acquireReadUncached  -> grantReadUncached,
+        acquireReadExclusive -> grantReadExclusive,                                            
-      acquireWriteUncached -> grantWriteUncached,
+        acquireInvalidateOthers -> grantReadExclusiveAck  //TODO: add this to MESI for broadcast?
-      acquireReadWordUncached  -> grantReadWordUncached,
+      )))
-      acquireWriteWordUncached -> grantWriteWordUncached,
-      acquireAtomicUncached -> grantAtomicUncached,
-      acquireInvalidateOthers -> grantReadExclusiveAck  //TODO: add this to MESI?
-    ))
   }
-  def getGrantType(r: Release, m: MasterMetadata): UInt = {
+  def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck
-    MuxLookup(r.r_type, grantReadUncached, Array(
-      releaseVoluntaryInvalidateData -> grantVoluntaryAck
-    ))
-  }
-
 
   def getProbeType(a: Acquire, m: MasterMetadata): UInt = {
-    MuxLookup(a.a_type, probeCopy, Array(
+    Mux(a.uncached, 
-      acquireReadShared -> probeDowngrade,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireReadExclusive -> probeInvalidate, 
+        Acquire.uncachedRead -> probeCopy, 
-      acquireReadUncached -> probeCopy, 
+        Acquire.uncachedWrite -> probeInvalidate,
-      acquireWriteUncached -> probeInvalidate,
+        Acquire.uncachedAtomic -> probeInvalidate
-      acquireReadWordUncached -> probeCopy, 
+      )),
-      acquireWriteWordUncached -> probeInvalidate,
+      MuxLookup(a.a_type, probeCopy, Array(
-      acquireAtomicUncached -> probeInvalidate,
+        acquireReadShared -> probeDowngrade,
-      acquireInvalidateOthers -> probeInvalidateOthers
+        acquireReadExclusive -> probeInvalidate, 
-    ))
+        acquireInvalidateOthers -> probeInvalidateOthers
+      )))
   }
   def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate
 
-  def requiresOuterRead(a_type: UInt) = {
+  def requiresOuterRead(acq: Acquire, m: MasterMetadata) = 
-      (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers)
+    Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers)
-  }
+  def requiresOuterWrite(acq: Acquire, m: MasterMetadata) =
-  def requiresOuterWrite(a_type: UInt) = {
+    Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false))
-      (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached)
-  }
 
-  def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck
+  def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck
-  def requiresAckForRelease(r_type: UInt) = Bool(false)
+  def requiresAckForRelease(r: Release) = Bool(false)
-  def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached
+  def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead
-  def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers)
+  def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers)
 
   def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData)
 }

uncore/src/main/scala/consts.scala

@@ -16,7 +16,9 @@ trait MemoryOpConstants {
   val MT_BU = Bits("b100")
   val MT_HU = Bits("b101")
   val MT_WU = Bits("b110")
+  val MT_CB = Bits("b111") // cache block
 
+  val NUM_XA_OPS = 9
   val M_SZ      = 5
   val M_X       = Bits("b?????");
   val M_XRD     = Bits("b00000"); // int load

uncore/src/main/scala/htif.scala

@@ -135,7 +135,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
     mem_acked := Bool(true)
     mem_gxid := io.mem.grant.bits.payload.master_xact_id
     mem_gsrc := io.mem.grant.bits.header.src
-    mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type)
+    mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload)
   }
   io.mem.grant.ready := Bool(true)
 
@@ -193,8 +193,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters {
   acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq
   val init_addr = addr.toUInt >> UInt(offsetBits-3)
   acq_q.io.enq.bits := Mux(cmd === cmd_writemem, 
-    Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), 
+    UncachedWrite(init_addr, UInt(0)), 
-    Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0)))
+    UncachedRead(init_addr))
   io.mem.acquire.valid := acq_q.io.deq.valid
   acq_q.io.deq.ready := io.mem.acquire.ready
   io.mem.acquire.bits.payload := acq_q.io.deq.bits

uncore/src/main/scala/memserdes.scala

@@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module {
   // Aggregate incoming MemIO responses into TL Grants
   io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max)
   io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max))
-  io.uncached.grant.bits.payload := Grant(co.getUncachedReadGrantType, tag_in, UInt(0), buf_in)
+  io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), buf_in)
   when(!active_in && io.mem.resp.valid) {
     active_in := Bool(true)
     cnt_in := UInt(1)

uncore/src/main/scala/tilelink.scala

@@ -2,65 +2,90 @@
 
 package uncore
 import Chisel._
+import scala.math.max
 
 case object TLId extends Field[String]
-case object TLCoherence extends Field[CoherencePolicyWithUncached]
+case object TLCoherence extends Field[CoherencePolicy]
 case object TLAddrBits extends Field[Int]
 case object TLMasterXactIdBits extends Field[Int]
 case object TLClientXactIdBits extends Field[Int]
 case object TLDataBits extends Field[Int]
-case object TLWriteMaskBits extends Field[Int]
-case object TLWordAddrBits extends Field[Int]
-case object TLAtomicOpBits extends Field[Int]
 
-trait HasPhysicalAddress extends Bundle {
+abstract trait TileLinkParameters extends UsesParameters {
-  val addr = UInt(width = params(TLAddrBits))
+  val tlAddrBits = params(TLAddrBits)
+  val tlClientXactIdBits = params(TLClientXactIdBits)
+  val tlMasterXactIdBits = params(TLMasterXactIdBits)
+  val tlDataBits = params(TLDataBits)
+  val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits
+  val tlSubblockAddrBits = log2Up(tlWriteMaskBits)
+  val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS)
+  val tlUncachedOperandSizeBits = MT_SZ
+  val tlSubblockUnionBits = max(tlWriteMaskBits, 
+                             (tlSubblockAddrBits + 
+                                tlUncachedOperandSizeBits + 
+                                tlAtomicOpcodeBits))
 }
 
-trait HasClientTransactionId extends Bundle {
+class TLBundle extends Bundle with TileLinkParameters
-  val client_xact_id = Bits(width = params(TLClientXactIdBits))
+
+trait HasPhysicalAddress extends TLBundle {
+  val addr = UInt(width = tlAddrBits)
 }
 
-trait HasMasterTransactionId extends Bundle {
+trait HasClientTransactionId extends TLBundle {
-  val master_xact_id = Bits(width = params(TLMasterXactIdBits))
+  val client_xact_id = Bits(width = tlClientXactIdBits)
 }
 
-trait HasTileLinkData extends Bundle {
+trait HasMasterTransactionId extends TLBundle {
-  val data = Bits(width = params(TLDataBits))
+  val master_xact_id = Bits(width = tlMasterXactIdBits)
 }
 
-trait SourcedMessage extends Bundle
+trait HasTileLinkData extends TLBundle {
+  val data = UInt(width = tlDataBits)
+}
+
+trait SourcedMessage extends TLBundle
 trait ClientSourcedMessage extends SourcedMessage
 trait MasterSourcedMessage extends SourcedMessage
 
-object Acquire 
+class Acquire extends ClientSourcedMessage 
-{
+    with HasPhysicalAddress 
-  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = {
+    with HasClientTransactionId 
+    with HasTileLinkData {
+  val uncached = Bool()
+  val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth))
+  val subblock =  Bits(width = tlSubblockUnionBits)
+  val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits
+  val opSzOff = tlUncachedOperandSizeBits + sbAddrOff
+  def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0)
+  def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits)
+  def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff)
+  def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0)
+  def is(t: UInt) = a_type === t
+}
+
+object Acquire {
+  val nUncachedAcquireTypes = 3
+  //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes)
+  def uncachedRead = UInt(0)
+  def uncachedWrite = UInt(1)
+  def uncachedAtomic = UInt(2)
+  def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type)
+  def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite
+  def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite
+
+  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
     val acq = new Acquire
+    acq.uncached := Bool(false)
     acq.a_type := a_type
     acq.addr := addr
     acq.client_xact_id := client_xact_id
-    acq.data := Bits(0)
-    acq.write_mask := Bits(0)
-    acq.subword_addr := Bits(0)
-    acq.atomic_opcode := Bits(0)
-    acq
-  }
-  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
-    val acq = apply(a_type, addr, client_xact_id)
     acq.data := data
+    acq.subblock := UInt(0)
     acq
   }
-  def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = {
+  def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = {
-    val acq = apply(a_type, addr, client_xact_id, data)
+    apply(a_type, addr, client_xact_id, UInt(0))
-    acq.write_mask := write_mask
-    acq
-  }
-  def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = {
-    val acq = apply(a_type, addr, client_xact_id, data)
-    acq.subword_addr := subword_addr
-    acq.atomic_opcode := atomic_opcode
-    acq
   }
   def apply(a: Acquire): Acquire = {
     val acq = new Acquire
@@ -69,94 +94,117 @@ object Acquire
   }
 }
 
-class Acquire extends ClientSourcedMessage 
+object UncachedRead {
-    with HasPhysicalAddress 
+  def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = {
-    with HasClientTransactionId 
+    val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id)
-    with HasTileLinkData {
+    acq.uncached := Bool(true)
-  val a_type = UInt(width = params(TLCoherence).acquireTypeWidth)
+    acq.subblock := Cat(subblock_addr, operand_sz)
-  val write_mask = Bits(width = params(TLWriteMaskBits))
+    acq
-  val subword_addr = Bits(width = params(TLWordAddrBits))
+  }
-  val atomic_opcode = Bits(width = params(TLAtomicOpBits))
+  def apply(addr: UInt, client_xact_id: UInt): Acquire = {
-  def is(t: UInt) = a_type === t
+    apply(addr, client_xact_id, UInt(0), MT_CB)
+  }
+  def apply(addr: UInt): Acquire = {
+    apply(addr, UInt(0), UInt(0), MT_CB)
+  }
 }
 
-object Probe
+object UncachedWrite {
-{
+  def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = {
-  def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = {
+    val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data)
+    acq.uncached := Bool(true)
+    acq.subblock := write_mask
+    acq
+  }
+  def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = {
+    apply(addr, client_xact_id, SInt(-1), data)
+  }
+  def apply(addr: UInt, data: UInt): Acquire = {
+    apply(addr, UInt(0), data)
+  }
+}
+
+object UncachedAtomic {
+  def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, 
+      subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = {
+    val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data)
+    acq.uncached := Bool(true)
+    acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz)
+    acq
+  }
+}
+
+object Probe {
+  def apply(p_type: UInt, addr: UInt) = {
     val prb = new Probe
     prb.p_type := p_type
     prb.addr := addr
-    prb.master_xact_id := master_xact_id
     prb
   }
 }
 
 class Probe extends MasterSourcedMessage 
-    with HasPhysicalAddress 
+    with HasPhysicalAddress {
-    with HasMasterTransactionId {
   val p_type = UInt(width = params(TLCoherence).probeTypeWidth)
   def is(t: UInt) = p_type === t
 }
 
-object Release
+object Release {
-{
+  def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = {
-  def apply(r_type: UInt, addr: UInt, data: UInt): Release = {
-    val rel = new Release
-    rel.r_type := r_type
-    rel.addr := addr
-    rel.data := data
-    rel
-  }
-  def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = {
     val rel = new Release
     rel.r_type := r_type
     rel.addr := addr
     rel.client_xact_id := client_xact_id
-    rel.master_xact_id := master_xact_id
-    rel.data := UInt(0)
-    rel
-  }
-  def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = {
-    val rel = apply(r_type, addr, client_xact_id, master_xact_id)
     rel.data := data
     rel
   }
+  def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = {
+    apply(r_type, addr, client_xact_id, UInt(0))
+  }
+  def apply(r_type: UInt, addr: UInt): Release = {
+    apply(r_type, addr, UInt(0), UInt(0))
+  }
 }
 
 class Release extends ClientSourcedMessage 
     with HasPhysicalAddress 
     with HasClientTransactionId 
-    with HasMasterTransactionId 
     with HasTileLinkData {
   val r_type = UInt(width = params(TLCoherence).releaseTypeWidth)
   def is(t: UInt) = r_type === t
 }
 
-object Grant
-{
-  def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = {
-    val gnt = new Grant
-    gnt.g_type := g_type
-    gnt.client_xact_id := client_xact_id
-    gnt.master_xact_id := master_xact_id
-    gnt.data := UInt(0)
-    gnt
-  }
-  def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = {
-    val gnt = apply(g_type, client_xact_id, master_xact_id)
-    gnt.data := data
-    gnt
-  }
-}
-
 class Grant extends MasterSourcedMessage 
     with HasTileLinkData 
     with HasClientTransactionId 
     with HasMasterTransactionId {
-  val g_type = UInt(width = params(TLCoherence).grantTypeWidth)
+  val uncached = Bool()
+  val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth))
   def is(t: UInt) = g_type === t
 }
 
+object Grant {
+  val nUncachedGrantTypes = 3
+  //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes)
+  def uncachedRead = UInt(0)
+  def uncachedWrite = UInt(1)
+  def uncachedAtomic = UInt(2)
+  def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type)
+
+  def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = {
+    val gnt = new Grant
+    gnt.uncached := uncached
+    gnt.g_type := g_type
+    gnt.client_xact_id := client_xact_id
+    gnt.master_xact_id := master_xact_id
+    gnt.data := data
+    gnt
+  }
+  def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = {
+    apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0))
+  }
+}
+
 class Finish extends ClientSourcedMessage with HasMasterTransactionId
 
 

uncore/src/main/scala/uncore.scala

@@ -5,14 +5,17 @@ import Chisel._
 
 case object NReleaseTransactors extends Field[Int]
 case object NAcquireTransactors extends Field[Int]
+case object L2StoreDataQueueDepth extends Field[Int]
 case object NClients extends Field[Int]
 
 abstract trait CoherenceAgentParameters extends UsesParameters {
   val co = params(TLCoherence)
-  val nReleaseTransactors = params(NReleaseTransactors)
+  val nReleaseTransactors = 1
   val nAcquireTransactors = params(NAcquireTransactors)
   val nTransactors = nReleaseTransactors + nAcquireTransactors
   val nClients = params(NClients)
+  val sdqDepth = params(L2StoreDataQueueDepth)
+  val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1
 }
 
 abstract class CoherenceAgent(innerId: String, outerId: String) extends Module
@@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module
 class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends 
     CoherenceAgent(innerId, outerId) {
 
+  // Queue to store impending UncachedWrite data
+  val sdq_val = Reg(init=Bits(0, sdqDepth))
+  val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0))
+  val sdq_rdy = !sdq_val.andR
+  val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload)
+  val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)}
+  when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data }
+
   // Create SHRs for outstanding transactions
   val trackerList = (0 until nReleaseTransactors).map(id => 
-    Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ 
+    Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ 
       (nReleaseTransactors until nTransactors).map(id => 
-        Module(new AcquireTracker(id, bankId, innerId, outerId)))
+        Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits}))
   
   // Propagate incoherence flags
   trackerList.map(_.io.tile_incoherent := io.incoherent.toBits)
@@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends
     val t = trackerList(i).io.inner
     alloc_arb.io.in(i).valid := t.acquire.ready
     t.acquire.bits := acquire.bits
+    t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1))
     t.acquire.valid := alloc_arb.io.in(i).ready
   }
-  acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires
+  acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires
-  alloc_arb.io.out.ready := acquire.valid && !block_acquires
+  alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires
 
   // Handle probe request generation
   val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size))
@@ -62,17 +74,24 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends
   val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_)
   val block_releases = Bool(false)
   val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b}
-  //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response
+  val release_idx = Mux(voluntary, UInt(0), conflict_idx)
-  val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id)
+  // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response
   for( i <- 0 until trackerList.size ) {
     val t = trackerList(i).io.inner
     t.release.bits := release.bits 
+    t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else  UInt(0))
     t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases
   }
   release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases
 
+  val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) }
+  when(voluntary && release.fire()) {
+    vwbdq(release_idx) := release.bits.payload.data
+  }
+
   // Reply to initial requestor
   val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size))
+  io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data
   io.inner.grant <> grant_arb.io.out
   grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant }
 
@@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends
 
   // Create an arbiter for the one memory port
   val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size),
-                         {case TLId => outerId})
+                         { case TLId => outerId; case TLDataBits => sdqIdBits })
   outer_arb.io.in zip  trackerList map { case(arb, t) => arb <> t.io.outer }
+  val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0)
+  val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1)
+  val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1)
+  val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2)
+  val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq
+  io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id),
+                                              Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data))
   io.outer <> outer_arb.io.out
+
+  // Update SDQ valid bits
+  when (io.outer.acquire.valid || sdq_enq) {
+    sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | 
+               PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq)
+  }
 }
 
 
@@ -112,17 +144,16 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
   val state = Reg(init=s_idle)
   val xact  = Reg{ new Release }
   val init_client_id = Reg(init=UInt(0, width = log2Up(nClients)))
-  val incoming_rel = io.inner.release.bits
 
   io.has_acquire_conflict := Bool(false)
-  io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && 
+  io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && 
                                (state != s_idle)
 
   io.outer.grant.ready := Bool(false)
   io.outer.acquire.valid := Bool(false)
   io.outer.acquire.bits.header.src := UInt(bankId) 
   //io.outer.acquire.bits.header.dst TODO
-  io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType,
+  io.outer.acquire.bits.payload := Bundle(UncachedWrite(
                                             xact.addr,
                                             UInt(trackerId),
                                             xact.data),
@@ -133,17 +164,18 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute
   io.inner.grant.valid := Bool(false)
   io.inner.grant.bits.header.src := UInt(bankId)
   io.inner.grant.bits.header.dst := init_client_id
-  io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),
+  io.inner.grant.bits.payload := Grant(Bool(false),
+                                        co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush),
                                         xact.client_xact_id,
                                         UInt(trackerId))
 
   switch (state) {
     is(s_idle) {
-      io.inner.release.ready := Bool(true)
       when( io.inner.release.valid ) {
-        xact := incoming_rel.payload
+        io.inner.release.ready := Bool(true)
-        init_client_id := incoming_rel.header.src
+        xact := c_rel.payload
-        state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack)
+        init_client_id := c_rel.header.src
+        state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack)
       }
     }
     is(s_mem) {
@@ -169,15 +201,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
   val curr_p_id = PriorityEncoder(probe_flags)
 
   val pending_outer_write = co.messageHasData(xact)
-  val pending_outer_read = co.requiresOuterRead(xact.a_type)
+  val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush)
-  val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, 
+  val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data),
-                                       xact.addr, UInt(trackerId), xact.data),
+                          { case TLId => outerId })
-                                    { case TLId => outerId })
+  val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId
-  val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, 
+                          { case TLId => outerId })
-                                       xact.addr, UInt(trackerId), c_rel.payload.data),
+  val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)),
-                                    { case TLId => outerId })
+                      { case TLId => outerId })
-  val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)),
-                                    { case TLId => outerId })
 
   val probe_initial_flags = Bits(width = nClients)
   probe_initial_flags := Bits(0)
@@ -200,18 +230,16 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
   io.inner.probe.valid := Bool(false)
   io.inner.probe.bits.header.src := UInt(bankId)
   io.inner.probe.bits.header.dst := curr_p_id
-  io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush),
+  io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr)
-                                               xact.addr,
-                                               UInt(trackerId))
 
-  val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)
   io.inner.grant.valid := Bool(false)
   io.inner.grant.bits.header.src := UInt(bankId)
   io.inner.grant.bits.header.dst := init_client_id
-  io.inner.grant.bits.payload := Grant(grant_type,
+  io.inner.grant.bits.payload := Grant(xact.uncached,
+                                        co.getGrantType(xact, co.masterMetadataOnFlush),
                                         xact.client_xact_id,
                                         UInt(trackerId),
-                                        m_gnt.payload.data)
+                                        UInt(0)) // Data bypassed in parent
 
   io.inner.acquire.ready := Bool(false)
   io.inner.release.ready := Bool(false)
@@ -220,7 +248,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
     is(s_idle) {
       io.inner.acquire.ready := Bool(true)
       val needs_outer_write = co.messageHasData(c_acq.payload)
-      val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type)
+      val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush)
       when( io.inner.acquire.valid ) {
         xact := c_acq.payload
         init_client_id := c_acq.header.src
@@ -268,7 +296,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
       io.outer.acquire.valid := Bool(true)
       io.outer.acquire.bits.payload := outer_read
       when(io.outer.acquire.ready) {
-        state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle)
+        state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle)
       }
     }
     is(s_mem_write) {
@@ -281,7 +309,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri
     is(s_make_grant) {
       io.inner.grant.valid := Bool(true)
       when(io.inner.grant.ready) { 
-        state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle)
+        state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle)
       }
     }
     is(s_busy) { // Nothing left to do but wait for transaction to complete