uncore and rocket changes for new xact types

rocket/src/main/scala/coherence.scala

@@ -3,10 +3,39 @@ package rocket
 import Chisel._
 import Constants._
 
-class TransactionInit extends Bundle {
+object TransactionInit 
+{
+  def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = {
+    val init = new TransactionInit
+    init.x_type := x_type
+    init.addr := addr
+    init.tile_xact_id := tile_xact_id
+    init
+  }
+  def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = {
+    val init = new TransactionInit
+    init.x_type := x_type
+    init.addr := addr
+    init.tile_xact_id := tile_xact_id
+    init.write_mask := write_mask
+    init
+  }
+  def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = {
+    val init = new TransactionInit
+    init.x_type := x_type
+    init.addr := addr
+    init.tile_xact_id := tile_xact_id
+    init.subword_addr := subword_addr
+    init.atomic_opcode := atomic_opcode
+    init
+  }
+}
+class TransactionInit extends PhysicalAddress {
   val x_type = Bits(width = X_INIT_TYPE_MAX_BITS)
   val tile_xact_id = Bits(width = TILE_XACT_ID_BITS)
-  val address = UFix(width = PADDR_BITS - OFFSET_BITS)
+  val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS)
+  val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS)
+  val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS)
 }
 
 class TransactionInitData extends MemData
@@ -15,10 +44,9 @@ class TransactionAbort extends Bundle {
   val tile_xact_id = Bits(width = TILE_XACT_ID_BITS)
 }
 
-class ProbeRequest extends Bundle {
+class ProbeRequest extends PhysicalAddress {
   val p_type = Bits(width = P_REQ_TYPE_MAX_BITS)
   val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS)
-  val address = Bits(width = PADDR_BITS - OFFSET_BITS)
 }
 
 class ProbeReply extends Bundle {
@@ -76,6 +104,7 @@ abstract class CoherencePolicy {
   def messageHasData (init: TransactionInit): Bool
   def messageHasData (reply: TransactionReply): Bool
   def messageUpdatesDataArray (reply: TransactionReply): Bool
+  def messageIsUncached(init: TransactionInit): Bool
 
   def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool
   def getTransactionReplyType(x_type: UFix, count: UFix): Bits
@@ -86,8 +115,11 @@ abstract class CoherencePolicy {
 }
 
 trait UncachedTransactions {
-  def getTransactionInitTypeOnUncachedRead(): UFix
-  def getTransactionInitTypeOnUncachedWrite(): UFix
+  def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit
+  def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit
+  def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit
+  def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit
+  def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit
 }
 
 abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions
@@ -115,6 +147,8 @@ class ThreeStateIncoherence extends IncoherentPolicy {
   val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() }
   val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() }
   val probeRepInvalidateAck :: Nil = Enum(1){ UFix() }
+  val uncachedTypeList = List() 
+  val hasDataTypeList = List(xactInitWriteback)
 
   def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty)
   def isValid (state: UFix): Bool = state != tileInvalid
@@ -149,9 +183,10 @@ class ThreeStateIncoherence extends IncoherentPolicy {
   def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO
   def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback
 
-  def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback)
+  def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_)
   def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData)
   def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData)
+  def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_)
 }
 
 class MICoherence extends CoherencePolicyWithUncached {
@@ -159,10 +194,12 @@ class MICoherence extends CoherencePolicyWithUncached {
   val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() }
   val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() }
 
-  val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() }
-  val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } 
+  val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() }
+  val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() }
   val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() }
   val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() }
+  val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
+  val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
 
   def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid
   def isValid (state: UFix): Bool = state != tileInvalid
@@ -191,7 +228,10 @@ class MICoherence extends CoherencePolicyWithUncached {
     MuxLookup(incoming.x_type, tileInvalid, Array(
       xactReplyReadExclusive -> tileValid,
       xactReplyReadUncached  -> tileInvalid,
-      xactReplyWriteUncached -> tileInvalid
+      xactReplyWriteUncached -> tileInvalid,
+      xactReplyReadWordUncached -> tileInvalid,
+      xactReplyWriteWordUncached -> tileInvalid,
+      xactReplyAtomicUncached -> tileInvalid
     ))
   } 
   def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = {
@@ -201,8 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached {
     ))
   }
 
-  def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached 
-  def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached 
+  def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id)
+  def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id)
+  def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id)
+  def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask)
+  def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op)
+
   def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive
   def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive
   def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached
@@ -227,15 +271,14 @@ class MICoherence extends CoherencePolicyWithUncached {
     (reply.p_type === probeRepInvalidateData ||
      reply.p_type === probeRepCopyData)
   }
-  def messageHasData (init: TransactionInit): Bool = {
-    (init.x_type === xactInitWriteUncached)
-  }
+  def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_)
   def messageHasData (reply: TransactionReply): Bool = {
-    (reply.x_type != xactReplyWriteUncached)
+    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached)
   }
   def messageUpdatesDataArray (reply: TransactionReply): Bool = {
     (reply.x_type === xactReplyReadExclusive)
   }
+  def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_)
 
   def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2)
 
@@ -243,7 +286,10 @@ class MICoherence extends CoherencePolicyWithUncached {
     MuxLookup(x_type, xactReplyReadUncached, Array(
       xactInitReadExclusive -> xactReplyReadExclusive,
       xactInitReadUncached  -> xactReplyReadUncached,
-      xactInitWriteUncached -> xactReplyWriteUncached
+      xactInitWriteUncached -> xactReplyWriteUncached,
+      xactInitReadWordUncached  -> xactReplyReadWordUncached,
+      xactInitWriteWordUncached -> xactReplyWriteWordUncached,
+      xactInitAtomicUncached -> xactReplyAtomicUncached
     ))
   }
 
@@ -251,7 +297,10 @@ class MICoherence extends CoherencePolicyWithUncached {
     MuxLookup(x_type, probeReqCopy, Array(
       xactInitReadExclusive -> probeReqInvalidate, 
       xactInitReadUncached -> probeReqCopy, 
-      xactInitWriteUncached -> probeReqInvalidate
+      xactInitWriteUncached -> probeReqInvalidate,
+      xactInitReadWordUncached -> probeReqCopy, 
+      xactInitWriteWordUncached -> probeReqInvalidate,
+      xactInitAtomicUncached -> probeReqInvalidate
     ))
   }
 
@@ -271,17 +320,19 @@ class MEICoherence extends CoherencePolicyWithUncached {
   val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() }
   val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() }
 
-  val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() }
-  val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } 
+  val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() }
+  val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() }
   val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() }
   val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() }
+  val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
+  val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
 
   def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid
   def isValid (state: UFix): Bool = state != tileInvalid
 
   def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = {
     val (read, write) = cpuCmdToRW(cmd)
-    (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) ||
+    (read && messageIsUncached(outstanding)) ||
       (write && (outstanding.x_type != xactInitReadExclusiveDirty))
   }
   def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = {
@@ -311,7 +362,10 @@ class MEICoherence extends CoherencePolicyWithUncached {
       xactReplyReadExclusive  -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean),
       xactReplyReadExclusiveAck -> tileExclusiveDirty, 
       xactReplyReadUncached -> tileInvalid,
-      xactReplyWriteUncached -> tileInvalid
+      xactReplyWriteUncached -> tileInvalid,
+      xactReplyReadWordUncached -> tileInvalid,
+      xactReplyWriteWordUncached -> tileInvalid,
+      xactReplyAtomicUncached -> tileInvalid
     ))
   } 
   def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = {
@@ -322,8 +376,12 @@ class MEICoherence extends CoherencePolicyWithUncached {
     ))
   }
 
-  def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached 
-  def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached 
+  def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id)
+  def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id)
+  def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id)
+  def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask)
+  def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op)
+
   def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
     val (read, write) = cpuCmdToRW(cmd)
     Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean)
@@ -357,15 +415,14 @@ class MEICoherence extends CoherencePolicyWithUncached {
      reply.p_type === probeRepDowngradeData ||
      reply.p_type === probeRepCopyData)
   }
-  def messageHasData (init: TransactionInit): Bool = {
-    (init.x_type === xactInitWriteUncached)
-  }
+  def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_)
   def messageHasData (reply: TransactionReply): Bool = {
-    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck)
+    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached)
   }
   def messageUpdatesDataArray (reply: TransactionReply): Bool = {
     (reply.x_type === xactReplyReadExclusive)
   }
+  def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_)
 
   def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2)
 
@@ -374,7 +431,10 @@ class MEICoherence extends CoherencePolicyWithUncached {
       xactInitReadExclusiveClean -> xactReplyReadExclusive,
       xactInitReadExclusiveDirty -> xactReplyReadExclusive,
       xactInitReadUncached  -> xactReplyReadUncached,
-      xactInitWriteUncached -> xactReplyWriteUncached
+      xactInitWriteUncached -> xactReplyWriteUncached,
+      xactInitReadWordUncached  -> xactReplyReadWordUncached,
+      xactInitWriteWordUncached -> xactReplyWriteWordUncached,
+      xactInitAtomicUncached -> xactReplyAtomicUncached
     ))
   }
 
@@ -383,7 +443,10 @@ class MEICoherence extends CoherencePolicyWithUncached {
       xactInitReadExclusiveClean -> probeReqInvalidate,
       xactInitReadExclusiveDirty -> probeReqInvalidate, 
       xactInitReadUncached -> probeReqCopy, 
-      xactInitWriteUncached -> probeReqInvalidate
+      xactInitWriteUncached -> probeReqInvalidate,
+      xactInitReadWordUncached -> probeReqCopy, 
+      xactInitWriteWordUncached -> probeReqInvalidate,
+      xactInitAtomicUncached -> probeReqInvalidate
     ))
   }
 
@@ -403,10 +466,12 @@ class MSICoherence extends CoherencePolicyWithUncached {
   val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() }
   val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() }
 
-  val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() }
-  val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } 
+  val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() }
+  val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() }
   val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() }
   val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() }
+  val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
+  val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
 
   def isHit (cmd: Bits, state: UFix): Bool = {
     val (read, write) = cpuCmdToRW(cmd)
@@ -419,7 +484,7 @@ class MSICoherence extends CoherencePolicyWithUncached {
 
   def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = {
     val (read, write) = cpuCmdToRW(cmd)
-    (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) ||
+    (read && messageIsUncached(outstanding)) || 
       (write && (outstanding.x_type != xactInitReadExclusive))
   }
   def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = {
@@ -450,7 +515,10 @@ class MSICoherence extends CoherencePolicyWithUncached {
       xactReplyReadExclusive  -> tileExclusiveDirty,
       xactReplyReadExclusiveAck -> tileExclusiveDirty, 
       xactReplyReadUncached -> tileInvalid,
-      xactReplyWriteUncached -> tileInvalid
+      xactReplyWriteUncached -> tileInvalid,
+      xactReplyReadWordUncached -> tileInvalid,
+      xactReplyWriteWordUncached -> tileInvalid,
+      xactReplyAtomicUncached -> tileInvalid
     ))
   } 
   def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = {
@@ -461,8 +529,12 @@ class MSICoherence extends CoherencePolicyWithUncached {
     ))
   }
 
-  def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached 
-  def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached 
+  def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id)
+  def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id)
+  def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id)
+  def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask)
+  def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op)
+
   def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
     val (read, write) = cpuCmdToRW(cmd)
     Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared)
@@ -496,15 +568,14 @@ class MSICoherence extends CoherencePolicyWithUncached {
      reply.p_type === probeRepDowngradeData ||
      reply.p_type === probeRepCopyData)
   }
-  def messageHasData (init: TransactionInit): Bool = {
-    (init.x_type === xactInitWriteUncached)
-  }
+  def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_)
   def messageHasData (reply: TransactionReply): Bool = {
-    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck)
+    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached)
   }
   def messageUpdatesDataArray (reply: TransactionReply): Bool = {
     (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive)
   }
+  def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_)
 
   def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2)
 
@@ -513,7 +584,10 @@ class MSICoherence extends CoherencePolicyWithUncached {
       xactInitReadShared    -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive),
       xactInitReadExclusive -> xactReplyReadExclusive,
       xactInitReadUncached  -> xactReplyReadUncached,
-      xactInitWriteUncached -> xactReplyWriteUncached
+      xactInitWriteUncached -> xactReplyWriteUncached,
+      xactInitReadWordUncached  -> xactReplyReadWordUncached,
+      xactInitWriteWordUncached -> xactReplyWriteWordUncached,
+      xactInitAtomicUncached -> xactReplyAtomicUncached
     ))
   }
 
@@ -542,10 +616,12 @@ class MESICoherence extends CoherencePolicyWithUncached {
   val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() }
   val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() }
 
-  val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() }
-  val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } 
+  val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() }
+  val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() }
   val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() }
   val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() }
+  val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
+  val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) 
 
   def isHit (cmd: Bits, state: UFix): Bool = {
     val (read, write) = cpuCmdToRW(cmd)
@@ -558,7 +634,7 @@ class MESICoherence extends CoherencePolicyWithUncached {
 
   def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = {
     val (read, write) = cpuCmdToRW(cmd)
-    (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) ||
+    (read && messageIsUncached(outstanding)) ||
       (write && (outstanding.x_type != xactInitReadExclusive))
   }
   def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = {
@@ -589,7 +665,10 @@ class MESICoherence extends CoherencePolicyWithUncached {
       xactReplyReadExclusive  -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean),
       xactReplyReadExclusiveAck -> tileExclusiveDirty, 
       xactReplyReadUncached -> tileInvalid,
-      xactReplyWriteUncached -> tileInvalid
+      xactReplyWriteUncached -> tileInvalid,
+      xactReplyReadWordUncached -> tileInvalid,
+      xactReplyWriteWordUncached -> tileInvalid,
+      xactReplyAtomicUncached -> tileInvalid
     ))
   } 
   def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = {
@@ -600,8 +679,12 @@ class MESICoherence extends CoherencePolicyWithUncached {
     ))
   }
 
-  def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached 
-  def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached 
+  def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id)
+  def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id)
+  def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id)
+  def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask)
+  def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op)
+
   def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
     val (read, write) = cpuCmdToRW(cmd)
     Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared)
@@ -635,15 +718,14 @@ class MESICoherence extends CoherencePolicyWithUncached {
      reply.p_type === probeRepDowngradeData ||
      reply.p_type === probeRepCopyData)
   }
-  def messageHasData (init: TransactionInit): Bool = {
-    (init.x_type === xactInitWriteUncached)
-  }
+  def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_)
   def messageHasData (reply: TransactionReply): Bool = {
-    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck)
+    (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached)
   }
   def messageUpdatesDataArray (reply: TransactionReply): Bool = {
     (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive)
   }
+  def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_)
 
   def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2)
 
@@ -652,7 +734,10 @@ class MESICoherence extends CoherencePolicyWithUncached {
       xactInitReadShared    -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive),
       xactInitReadExclusive -> xactReplyReadExclusive,
       xactInitReadUncached  -> xactReplyReadUncached,
-      xactInitWriteUncached -> xactReplyWriteUncached
+      xactInitWriteUncached -> xactReplyWriteUncached,
+      xactInitReadWordUncached  -> xactReplyReadWordUncached,
+      xactInitWriteWordUncached -> xactReplyWriteWordUncached,
+      xactInitAtomicUncached -> xactReplyAtomicUncached
     ))
   }
 
@@ -661,7 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached {
       xactInitReadShared -> probeReqDowngrade,
       xactInitReadExclusive -> probeReqInvalidate, 
       xactInitReadUncached -> probeReqCopy, 
-      xactInitWriteUncached -> probeReqInvalidate
+      xactInitWriteUncached -> probeReqInvalidate,
+      xactInitReadWordUncached -> probeReqCopy, 
+      xactInitWriteWordUncached -> probeReqInvalidate,
+      xactInitAtomicUncached -> probeReqInvalidate
     ))
   }
 

rocket/src/main/scala/consts.scala

@@ -200,6 +200,9 @@ object Constants
   val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS)
 
   val X_INIT_TYPE_MAX_BITS = 2
+  val X_INIT_WRITE_MASK_BITS = OFFSET_BITS
+  val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS)
+  val X_INIT_ATOMIC_OP_BITS = 4
   val X_REP_TYPE_MAX_BITS = 3
   val P_REQ_TYPE_MAX_BITS = 2
   val P_REP_TYPE_MAX_BITS = 3

rocket/src/main/scala/htif.scala

@@ -176,8 +176,8 @@ class rocketHTIF(w: Int, ncores: Int, co: CoherencePolicyWithUncached) extends C
     mem_req_data = Cat(packet_ram(idx), mem_req_data)
   }
   x_init.io.enq.valid := state === state_mem_req
-  x_init.io.enq.bits.x_type := Mux(cmd === cmd_writemem, co.getTransactionInitTypeOnUncachedWrite, co.getTransactionInitTypeOnUncachedRead)
-  x_init.io.enq.bits.address := addr.toUFix >> UFix(OFFSET_BITS-3)
+  val init_addr = addr.toUFix >> UFix(OFFSET_BITS-3)
+  x_init.io.enq.bits := Mux(cmd === cmd_writemem, co.getUncachedWriteTransactionInit(init_addr, UFix(0)), co.getUncachedReadTransactionInit(init_addr, UFix(0)))
   io.mem.xact_init <> x_init.io.deq
   io.mem.xact_init_data.valid:= state === state_mem_wdata
   io.mem.xact_init_data.bits.data := mem_req_data

rocket/src/main/scala/icache.scala

@@ -136,8 +136,7 @@ class rocketICache(sets: Int, assoc: Int, co: CoherencePolicyWithUncached) exten
   rdy := !io.cpu.itlb_miss && (state === s_ready) && (!r_cpu_req_val || tag_hit);
   io.cpu.resp_data := data_mux.io.out
   io.mem.xact_init.valid := (state === s_request) && finish_q.io.enq.ready
-  io.mem.xact_init.bits.x_type := co.getTransactionInitTypeOnUncachedRead
-  io.mem.xact_init.bits.address := r_cpu_miss_addr(tagmsb,indexlsb).toUFix
+  io.mem.xact_init.bits := co.getUncachedReadTransactionInit(r_cpu_miss_addr(tagmsb,indexlsb).toUFix, UFix(0))
   io.mem.xact_finish <> finish_q.io.deq
 
   // control state machine

rocket/src/main/scala/nbdcache.scala

@@ -282,7 +282,7 @@ class MSHR(id: Int, co: CoherencePolicy) extends Component {
 
   io.mem_req.valid := (state === s_refill_req) && !flush
   io.mem_req.bits.x_type := xacx_type
-  io.mem_req.bits.address := Cat(req.tag, req.idx).toUFix
+  io.mem_req.bits.addr := Cat(req.tag, req.idx).toUFix
   io.mem_req.bits.tile_xact_id := Bits(id)
   io.mem_finish <> finish_q.io.deq
 
@@ -475,7 +475,7 @@ class WritebackUnit(co: CoherencePolicy) extends Component {
 
   io.mem_req.valid := valid && !cmd_sent
   io.mem_req.bits.x_type := co.getTransactionInitTypeOnWriteback()
-  io.mem_req.bits.address := Cat(req.tag, req.idx).toUFix
+  io.mem_req.bits.addr := Cat(req.tag, req.idx).toUFix
   io.mem_req.bits.tile_xact_id := req.tile_xact_id
   io.mem_req_data.valid := data_req_fired && !is_probe
   io.mem_req_data.bits.data := io.data_resp
@@ -492,7 +492,7 @@ class ProbeUnit(co: CoherencePolicy) extends Component {
     val wb_req = (new FIFOIO) { new WritebackReq }
     val tag_match_way_oh = Bits(INPUT, NWAYS)
     val line_state = UFix(INPUT, 2)
-    val address = Bits(OUTPUT, PADDR_BITS-OFFSET_BITS)
+    val addr = Bits(OUTPUT, PADDR_BITS-OFFSET_BITS)
   }
 
   val s_reset :: s_invalid :: s_meta_req :: s_meta_resp :: s_mshr_req :: s_probe_rep :: s_writeback_req :: s_writeback_resp :: Nil = Enum(8) { UFix() }
@@ -535,16 +535,16 @@ class ProbeUnit(co: CoherencePolicy) extends Component {
   io.meta_req.valid := state === s_meta_req || state === s_meta_resp || state === s_mshr_req || state === s_probe_rep && hit
   io.meta_req.bits.way_en := Mux(state === s_probe_rep, way_oh, ~UFix(0, NWAYS))
   io.meta_req.bits.rw := state === s_probe_rep
-  io.meta_req.bits.idx := req.address
+  io.meta_req.bits.idx := req.addr
   io.meta_req.bits.data.state := co.newStateOnProbeRequest(req, line_state)
-  io.meta_req.bits.data.tag := req.address >> UFix(IDX_BITS)
+  io.meta_req.bits.data.tag := req.addr >> UFix(IDX_BITS)
   io.mshr_req.valid := state === s_meta_resp || state === s_mshr_req
-  io.address := req.address
+  io.addr := req.addr
 
   io.wb_req.valid := state === s_writeback_req
   io.wb_req.bits.way_oh := way_oh
-  io.wb_req.bits.idx := req.address
-  io.wb_req.bits.tag := req.address >> UFix(IDX_BITS)
+  io.wb_req.bits.idx := req.addr
+  io.wb_req.bits.tag := req.addr >> UFix(IDX_BITS)
 }
 
 class FlushUnit(lines: Int, co: CoherencePolicy) extends Component {
@@ -860,7 +860,7 @@ class HellaCache(co: CoherencePolicy) extends Component {
   meta_arb.io.in(3).bits.rw := Bool(false)
   meta_arb.io.in(3).bits.way_en := ~UFix(0, NWAYS)
   val early_tag_nack = !meta_arb.io.in(3).ready
-  val cpu_req_ppn = Mux(prober.io.mshr_req.valid, prober.io.address >> UFix(PGIDX_BITS-OFFSET_BITS), io.cpu.req.bits.ppn)
+  val cpu_req_ppn = Mux(prober.io.mshr_req.valid, prober.io.addr >> UFix(PGIDX_BITS-OFFSET_BITS), io.cpu.req.bits.ppn)
   val cpu_req_tag = Cat(cpu_req_ppn, r_cpu_req_idx)(tagmsb,taglsb)
   val tag_match_arr = (0 until NWAYS).map( w => co.isValid(meta.io.resp(w).state) && (meta.io.resp(w).tag === cpu_req_tag))
   val tag_match = Cat(Bits(0),tag_match_arr:_*).orR

rocket/src/main/scala/uncore.scala

@@ -3,14 +3,17 @@ package rocket
 import Chisel._
 import Constants._
 
+class PhysicalAddress extends Bundle {
+  val addr = UFix(width = PADDR_BITS - OFFSET_BITS)
+}
+
 class MemData extends Bundle {
   val data = Bits(width = MEM_DATA_BITS)
 }
 
-class MemReqCmd() extends Bundle 
+class MemReqCmd() extends PhysicalAddress
 {
   val rw = Bool()
-  val addr = UFix(width = PADDR_BITS - OFFSET_BITS)
   val tag = Bits(width = MEM_TAG_BITS)
 }
 
@@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component {
   io.probe_req.valid := Bool(false)
   io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0))
   io.probe_req.bits.global_xact_id  := UFix(id)
-  io.probe_req.bits.address := addr_
+  io.probe_req.bits.addr := addr_
   io.push_p_req      := Bits(0, width = ntiles)
   io.pop_p_rep       := Bits(0, width = ntiles)
   io.pop_p_rep_data  := Bits(0, width = ntiles)
@@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component {
   switch (state) {
     is(s_idle) {
       when( io.alloc_req.valid && io.can_alloc ) {
-        addr_ := io.alloc_req.bits.xact_init.address
+        addr_ := io.alloc_req.bits.xact_init.addr
         x_type_ := io.alloc_req.bits.xact_init.x_type
         init_tile_id_ := io.alloc_req.bits.tile_id
         tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id
@@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence)  extends CoherenceHub(1, co)
   io.mem.req_cmd.valid   := x_init.valid && !(is_write && io.mem.resp.valid)
   io.mem.req_cmd.bits.rw    := is_write
   io.mem.req_cmd.bits.tag   := x_init.bits.tile_xact_id
-  io.mem.req_cmd.bits.addr  := x_init.bits.address
+  io.mem.req_cmd.bits.addr  := x_init.bits.addr
   io.mem.req_data <> io.tiles(0).xact_init_data
 
   val x_rep = io.tiles(0).xact_rep
@@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH
     val conflicts = Vec(NGLOBAL_XACTS) { Bool() }
     for( i <- 0 until NGLOBAL_XACTS) {
       val t = trackerList(i).io
-      conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address)
+      conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr)
     }
     x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id
     want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits)))