WIP bugfixes: run until corrupted WB data (beats repeated)

src/main/scala/rocket/dcache.scala

@@ -283,7 +283,7 @@ class DCache(maxUncachedInFlight: Int = 2)(implicit val p: Parameters) extends L
     val a_address = s2_req.addr
     val a_size = s2_req.typ
     val a_data = Fill(beatWords, pstore1_storegen.data)
-    val acquire = edge.Acquire(a_source, a_address, lgCacheBlockBytes, s2_grow_param)._2 // TODO check cacheability
+    val acquire = edge.Acquire(a_source, a_address, lgCacheBlockBytes, s2_grow_param)._2 // TODO Cacheability already been checked?
     val get     = edge.Get(a_source, a_address, a_size)._2
     val put     = edge.Put(a_source, a_address, a_size, a_data)._2
     val atomics = if (edge.manager.anySupportLogical) {
@@ -305,9 +305,7 @@ class DCache(maxUncachedInFlight: Int = 2)(implicit val p: Parameters) extends L
 
     tl_out.a.valid := grantackq.io.enq.ready && ((s2_valid_cached_miss && !s2_victim_dirty) ||
                                           (s2_valid_uncached && !uncachedInFlight.asUInt.andR))
-    tl_out.a.bits := Mux(pstore1_amo && s2_write && s2_uncached, atomics,
+    tl_out.a.bits := Mux(!s2_uncached, acquire, Mux(!s2_write, get, Mux(!pstore1_amo, put, atomics)))
-                       Mux(s2_write && s2_uncached, put,
-                        Mux(s2_uncached, get, acquire)))
 
     // Set pending bits for outstanding TileLink transaction
     when (tl_out.a.fire()) {
@@ -392,13 +390,20 @@ class DCache(maxUncachedInFlight: Int = 2)(implicit val p: Parameters) extends L
     metaReadArb.io.in(1).bits.way_en := ~UInt(0, nWays)
 
     // release
-    val (writebackCount, writebackDone) = Counter(tl_out.c.fire() && inWriteback, refillCycles) //TODO firstlast?
+    val (_, c_last, c_address_inc) = edge.firstlast(tl_out.c)
-    val releaseDone = writebackDone || (tl_out.c.fire() && !inWriteback)
+    val releaseDone = tl_out.c.fire() && Mux(inWriteback, c_last, Bool(true))
     val releaseRejected = tl_out.c.valid && !tl_out.c.ready
     val s1_release_data_valid = Reg(next = dataArb.io.in(2).fire())
     val s2_release_data_valid = Reg(next = s1_release_data_valid && !releaseRejected)
+
+    // TODO refactor these counters
+    val (writebackCount, _) = Counter(tl_out.c.fire() && inWriteback, refillCycles)
     val releaseDataBeat = Cat(UInt(0), writebackCount) + Mux(releaseRejected, UInt(0), s1_release_data_valid + Cat(UInt(0), s2_release_data_valid))
 
+    val nackResponseMessage     = edge.ProbeAck(
+                                    b = probe_bits,
+                                    reportPermissions = TLPermissions.NtoN)
+
     val voluntaryReleaseMessage = edge.Release(
                                     fromSource = UInt(maxUncachedInFlight - 1),
                                     toAddress = probe_bits.address,
@@ -416,14 +421,14 @@ class DCache(maxUncachedInFlight: Int = 2)(implicit val p: Parameters) extends L
                                     data = s2_data))
 
     tl_out.c.valid := s2_release_data_valid
-    tl_out.c.bits := voluntaryReleaseMessage // TODO was ClientMetadata.onReset.makeRelease(probe_bits) ... s2_victim_state ok?
+    tl_out.c.bits := nackResponseMessage // TODO was ClientMetadata.onReset.makeRelease(probe_bits) ... ok?
     val newCoh = Wire(init = probeNewCoh)
     releaseWay := s2_probe_way
 
     when (s2_victimize && s2_victim_dirty) {
       assert(!(s2_valid && s2_hit_valid))
       release_state := s_voluntary_writeback
-      probe_bits.address := Cat(s2_victim_tag, s2_req.addr(idxMSB, idxLSB)) << rowOffBits
+      probe_bits.address := Cat(s2_victim_tag, s2_req.addr(idxMSB, idxLSB)) << idxLSB
     }
     when (s2_probe) {
       when (needs_vol_wb) { release_state := s_probe_rep_dirty }
@@ -456,7 +461,7 @@ class DCache(maxUncachedInFlight: Int = 2)(implicit val p: Parameters) extends L
 
     dataArb.io.in(2).valid := inWriteback && releaseDataBeat < refillCycles
     dataArb.io.in(2).bits.write := false
-    dataArb.io.in(2).bits.addr := tl_out.c.bits.address | (releaseDataBeat(log2Up(refillCycles)-1,0) << rowOffBits)
+    dataArb.io.in(2).bits.addr := tl_out.c.bits.address | c_address_inc
     dataArb.io.in(2).bits.way_en := ~UInt(0, nWays)
 
     metaWriteArb.io.in(2).valid := release_state.isOneOf(s_voluntary_write_meta, s_probe_write_meta)

src/main/scala/rocket/tile.scala

@@ -27,7 +27,7 @@ class RocketTile(tileId: Int)(implicit p: Parameters) extends LazyModule {
   val dcacheParams = p.alterPartial({
     case CacheName => "L1D"
     case TLId => "L1toL2"
-    case TileId => tileId // TODO using this messes with Heirarchical P&R
+    case TileId => tileId // TODO using this messes with Heirarchical P&R: change to io.hartid?
   })
   val icacheParams = p.alterPartial({
     case CacheName => "L1I"

src/main/scala/uncore/tilelink2/Broadcast.scala

@@ -96,7 +96,7 @@ class TLBroadcast(lineBytes: Int, numTrackers: Int = 4, bufferless: Boolean = fa
       d_normal.bits.param  := Mux(d_hasData, Mux(d_what(0), TLPermissions.toT, TLPermissions.toB), UInt(0))
     }
     d_normal.bits.sink := OHToUInt(d_trackerOH)
-    assert (!d_normal.valid || d_trackerOH.orR())
+    assert (!d_normal.valid || (d_trackerOH.orR() || d_normal.bits.opcode === TLMessages.ReleaseAck))
 
     // A tracker response is anything neither dropped nor a ReleaseAck
     val d_response = d_hasData || !d_what(1)

src/main/scala/uncore/tilelink2/Bundles.scala

@@ -53,53 +53,61 @@ object TLMessages
   */
 object TLPermissions
 {
+  val aWidth = 2
+  val bdWidth = 2
+  val cWidth = 3
+
   // Cap types (Grant = new permissions, Probe = permisions <= target)
-  val toT = UInt(0)
+  val toT = UInt(0, bdWidth)
-  val toB = UInt(1)
+  val toB = UInt(1, bdWidth)
-  val toN = UInt(2)
+  val toN = UInt(2, bdWidth)
   def isCap(x: UInt) = x <= toN
 
   // Grow types (Acquire = permissions >= target)
-  val NtoB = UInt(0)
+  val NtoB = UInt(0, aWidth)
-  val NtoT = UInt(1)
+  val NtoT = UInt(1, aWidth)
-  val BtoT = UInt(2)
+  val BtoT = UInt(2, aWidth)
   def isGrow(x: UInt) = x <= BtoT
 
   // Shrink types (ProbeAck, Release)
-  val TtoB = UInt(0)
+  val TtoB = UInt(0, cWidth)
-  val TtoN = UInt(1)
+  val TtoN = UInt(1, cWidth)
-  val BtoN = UInt(2)
+  val BtoN = UInt(2, cWidth)
   def isShrink(x: UInt) = x <= BtoN
 
   // Report types (ProbeAck)
-  val TtoT = UInt(3)
+  val TtoT = UInt(3, cWidth)
-  val BtoB = UInt(4)
+  val BtoB = UInt(4, cWidth)
-  val NtoN = UInt(5)
+  val NtoN = UInt(5, cWidth)
   def isReport(x: UInt) = x <= NtoN
 }
 
 object TLAtomics
 {
+  val width = 3 
+
   // Arithmetic types
-  val MIN  = UInt(0)
+  val MIN  = UInt(0, width)
-  val MAX  = UInt(1)
+  val MAX  = UInt(1, width)
-  val MINU = UInt(2)
+  val MINU = UInt(2, width)
-  val MAXU = UInt(3)
+  val MAXU = UInt(3, width)
-  val ADD  = UInt(4)
+  val ADD  = UInt(4, width)
   def isArithmetic(x: UInt) = x <= ADD
 
   // Logical types
-  val XOR  = UInt(0)
+  val XOR  = UInt(0, width)
-  val OR   = UInt(1)
+  val OR   = UInt(1, width)
-  val AND  = UInt(2)
+  val AND  = UInt(2, width)
-  val SWAP = UInt(3)
+  val SWAP = UInt(3, width)
   def isLogical(x: UInt) = x <= SWAP
 }
 
 object TLHints
 {
-  val PREFETCH_READ  = UInt(0)
+  val width = 1
-  val PREFETCH_WRITE = UInt(1)
+
+  val PREFETCH_READ  = UInt(0, width)
+  val PREFETCH_WRITE = UInt(1, width)
 }
 
 sealed trait TLChannel extends TLBundleBase {
@@ -114,7 +122,7 @@ final class TLBundleA(params: TLBundleParameters)
   val channelName = "'A' channel"
   // fixed fields during multibeat:
   val opcode  = UInt(width = 3)
-  val param   = UInt(width = 3) // amo_opcode || perms || hint
+  val param   = UInt(width = List(TLAtomics.width, TLPermissions.aWidth, TLHints.width).max) // amo_opcode || grow perms || hint
   val size    = UInt(width = params.sizeBits)
   val source  = UInt(width = params.sourceBits) // from
   val address = UInt(width = params.addressBits) // to
@@ -129,7 +137,7 @@ final class TLBundleB(params: TLBundleParameters)
   val channelName = "'B' channel"
   // fixed fields during multibeat:
   val opcode  = UInt(width = 3)
-  val param   = UInt(width = 3)
+  val param   = UInt(width = TLPermissions.bdWidth) // cap perms
   val size    = UInt(width = params.sizeBits)
   val source  = UInt(width = params.sourceBits) // to
   val address = UInt(width = params.addressBits) // from
@@ -144,7 +152,7 @@ final class TLBundleC(params: TLBundleParameters)
   val channelName = "'C' channel"
   // fixed fields during multibeat:
   val opcode  = UInt(width = 3)
-  val param   = UInt(width = 3)
+  val param   = UInt(width = TLPermissions.cWidth) // shrink or report perms
   val size    = UInt(width = params.sizeBits)
   val source  = UInt(width = params.sourceBits) // from
   val address = UInt(width = params.addressBits) // to
@@ -159,7 +167,7 @@ final class TLBundleD(params: TLBundleParameters)
   val channelName = "'D' channel"
   // fixed fields during multibeat:
   val opcode  = UInt(width = 3)
-  val param   = UInt(width = 2)
+  val param   = UInt(width = TLPermissions.bdWidth) // cap perms
   val size    = UInt(width = params.sizeBits)
   val source  = UInt(width = params.sourceBits) // to
   val sink    = UInt(width = params.sinkBits)   // from

src/main/scala/uncore/tilelink2/Edges.scala

@@ -205,7 +205,7 @@ class TLEdgeOut(
     a.size    := lgSize
     a.source  := fromSource
     a.address := toAddress
-    a.mask    := SInt(-1).asUInt
+    a.mask    := mask(toAddress, lgSize)
     a.data    := UInt(0)
     (legal, a)
   }
@@ -419,7 +419,7 @@ class TLEdgeIn(
     b.size    := lgSize
     b.source  := toSource
     b.address := fromAddress
-    b.mask    := SInt(-1).asUInt
+    b.mask    := mask(fromAddress, lgSize)
     b.data    := UInt(0)
     (legal, b)
   }

src/main/scala/uncore/tilelink2/Metadata.scala

@@ -10,10 +10,10 @@ import uncore.constants.MemoryOpConstants
 object ClientStates {
   val width = 2
 
-  val Nothing = UInt(0) 
+  val Nothing = UInt(0, width)
-  val Branch  = UInt(1)
+  val Branch  = UInt(1, width)
-  val Trunk   = UInt(2)
+  val Trunk   = UInt(2, width)
-  val Dirty   = UInt(3)
+  val Dirty   = UInt(3, width)
 
   def hasReadPermission(state: UInt): Bool = state > Nothing
   def hasWritePermission(state: UInt): Bool = state > Branch
@@ -24,7 +24,11 @@ object MemoryOpCategories extends MemoryOpConstants {
   val wi = Cat(Bool(false), Bool(true))  // Future op will write
   val rd = Cat(Bool(false), Bool(false)) // Op only reads
 
-  def categorize(cmd: UInt): UInt = Cat(isWrite(cmd), isWriteIntent(cmd))
+  def categorize(cmd: UInt): UInt = {
+    val cat = Cat(isWrite(cmd), isWriteIntent(cmd))
+    assert(cat.isOneOf(wr,wi,rd), "Could not categorize command.")
+    cat
+  }
 }
 
 /** Stores the client-side coherence information,
@@ -49,7 +53,8 @@ class ClientMetadata extends Bundle {
     import MemoryOpCategories._
     import TLPermissions._
     import ClientStates._
-    MuxTLookup(Cat(categorize(cmd), state), (Bool(false), UInt(0)), Seq(
+    val c = categorize(cmd)
+    MuxTLookup(Cat(c, state), (Bool(false), UInt(0)), Seq(
     //(effect, am now) -> (was a hit,   next)
       Cat(rd, Dirty)   -> (Bool(true),  Dirty),
       Cat(rd, Trunk)   -> (Bool(true),  Trunk),
@@ -71,7 +76,9 @@ class ClientMetadata extends Bundle {
     import MemoryOpCategories._
     import TLPermissions._
     import ClientStates._
-    MuxLookup(Cat(categorize(cmd), param), UInt(0), Seq(
+    val c = categorize(cmd)
+    assert(c === rd || param === toT, "Client was expecting trunk permissions.")   
+    MuxLookup(Cat(c, param), Nothing, Seq(
     //(effect param) -> (next)
       Cat(rd, toB)   -> Branch,
       Cat(rd, toT)   -> Trunk,
@@ -79,7 +86,6 @@ class ClientMetadata extends Bundle {
       Cat(wr, toT)   -> Dirty))
   }
 
-
   /** Does a secondary miss on the block require another Acquire message */
   def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = {
     import MemoryOpCategories._

src/main/scala/uncore/tilelink2/Monitor.scala

@@ -29,7 +29,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
       assert (bundle.size >= UInt(log2Ceil(edge.manager.beatBytes)), "'A' channel Acquire smaller than a beat" + extra)
       assert (is_aligned, "'A' channel Acquire address not aligned to size" + extra)
       assert (TLPermissions.isGrow(bundle.param), "'A' channel Acquire carries invalid grow param" + extra)
-      assert (~bundle.mask === UInt(0), "'A' channel Acquire contains invalid mask" + extra)
+      assert (bundle.mask === mask, "'A' channel Acquire contains invalid mask" + extra)
     }
 
     when (bundle.opcode === TLMessages.Get) {
@@ -94,7 +94,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
       assert (bundle.size >= UInt(log2Ceil(edge.manager.beatBytes)), "'B' channel Probe smaller than a beat" + extra)
       assert (is_aligned, "'B' channel Probe address not aligned to size" + extra)
       assert (TLPermissions.isCap(bundle.param), "'B' channel Probe carries invalid cap param" + extra)
-      assert (~bundle.mask === UInt(0).asUInt, "'B' channel Probe contains invalid mask" + extra)
+      assert (bundle.mask === mask, "'B' channel Probe contains invalid mask" + extra)
     }
 
     when (bundle.opcode === TLMessages.Get) {
@@ -102,7 +102,7 @@ class TLMonitor(gen: () => TLBundleSnoop, edge: () => TLEdge, sourceInfo: Source
       assert (address_ok, "'B' channel Get carries unmanaged address" + extra)
       assert (is_aligned, "'B' channel Get address not aligned to size" + extra)
       assert (bundle.param === UInt(0), "'B' channel Get carries invalid param" + extra)
-      assert (bundle.mask === mask, "'A' channel Get contains invalid mask" + extra)
+      assert (bundle.mask === mask, "'B' channel Get contains invalid mask" + extra)
     }
 
     when (bundle.opcode === TLMessages.PutFullData) {