Merge pull request #707 from ucb-bar/itim

ITIM

src/main/scala/coreplex/Configs.scala

@@ -19,6 +19,9 @@ class BaseCoreplexConfig extends Config ((site, here, up) => {
   case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */
   case ASIdBits => 0
   case XLen => 64 // Applies to all cores
+  case ResetVectorBits => site(PAddrBits)
+  case MaxHartIdBits => log2Up(site(NTiles))
+  case MaxPriorityLevels => 7
   case BuildCore => (p: Parameters) => new Rocket()(p)
   case RocketCrossing => SynchronousCrossing()
   case RocketTilesKey =>  Nil

src/main/scala/coreplex/Coreplex.scala

@@ -1,24 +0,0 @@
-// See LICENSE.SiFive for license details.
-
-package coreplex
-
-import Chisel._
-import config._
-import diplomacy._
-import util._
-
-/////
-
-class DefaultCoreplex(implicit p: Parameters) extends BaseCoreplex
-    with CoreplexRISCVPlatform
-    with HasRocketTiles {
-  override lazy val module = new DefaultCoreplexModule(this, () => new DefaultCoreplexBundle(this))
-}
-
-class DefaultCoreplexBundle[+L <: DefaultCoreplex](_outer: L) extends BaseCoreplexBundle(_outer)
-    with CoreplexRISCVPlatformBundle
-    with HasRocketTilesBundle
-
-class DefaultCoreplexModule[+L <: DefaultCoreplex, +B <: DefaultCoreplexBundle[L]](_outer: L, _io: () => B) extends BaseCoreplexModule(_outer, _io)
-    with CoreplexRISCVPlatformModule
-    with HasRocketTilesModule

src/main/scala/coreplex/RISCVPlatform.scala

@@ -3,19 +3,20 @@
 package coreplex
 
 import Chisel._
-import config._
+import config.Field
-import junctions._
 import diplomacy._
 import tile._
 import uncore.tilelink2._
 import uncore.devices._
 import util._
 
+case object MaxPriorityLevels extends Field[Int]
+
 trait CoreplexRISCVPlatform extends CoreplexNetwork {
   val module: CoreplexRISCVPlatformModule
 
   val debug = LazyModule(new TLDebugModule())
-  val plic  = LazyModule(new TLPLIC(maxPriorities = 7))
+  val plic  = LazyModule(new TLPLIC(maxPriorities = p(MaxPriorityLevels)))
   val clint = LazyModule(new CoreplexLocalInterrupter)
 
   debug.node := TLFragmenter(cbus_beatBytes, cbus_lineBytes)(cbus.node)
@@ -34,7 +35,7 @@ trait CoreplexRISCVPlatformBundle extends CoreplexNetworkBundle {
 
   val debug = new ClockedDMIIO().flip
   val rtcToggle = Bool(INPUT)
-  val resetVector = UInt(INPUT, p(XLen))
+  val resetVector = UInt(INPUT, p(ResetVectorBits))
   val ndreset = Bool(OUTPUT)
   val dmactive = Bool(OUTPUT)
 }

src/main/scala/coreplex/RocketPlex.scala

@@ -0,0 +1,20 @@
+// See LICENSE.SiFive for license details.
+
+package coreplex
+
+import Chisel._
+import config.Parameters
+
+class RocketPlex(implicit p: Parameters) extends BaseCoreplex
+    with CoreplexRISCVPlatform
+    with HasRocketTiles {
+  override lazy val module = new RocketPlexModule(this, () => new RocketPlexBundle(this))
+}
+
+class RocketPlexBundle[+L <: RocketPlex](_outer: L) extends BaseCoreplexBundle(_outer)
+    with CoreplexRISCVPlatformBundle
+    with HasRocketTilesBundle
+
+class RocketPlexModule[+L <: RocketPlex, +B <: RocketPlexBundle[L]](_outer: L, _io: () => B) extends BaseCoreplexModule(_outer, _io)
+    with CoreplexRISCVPlatformModule
+    with HasRocketTilesModule

src/main/scala/rocket/CSR.scala

@@ -153,7 +153,7 @@ class PerfCounterIO(implicit p: Parameters) extends CoreBundle
 class CSRFileIO(implicit p: Parameters) extends CoreBundle
     with HasRocketCoreParameters {
   val interrupts = new TileInterrupts().asInput
-  val hartid = UInt(INPUT, xLen)
+  val hartid = UInt(INPUT, hartIdLen)
   val rw = new Bundle {
     val addr = UInt(INPUT, CSR.ADDRSZ)
     val cmd = Bits(INPUT, CSR.SZ)

src/main/scala/rocket/DCache.scala

@@ -138,7 +138,8 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
     if (usingDataScratchpad) {
       metaWriteArb.io.out.ready := true
       metaReadArb.io.out.ready := !metaWriteArb.io.out.valid
-      val inScratchpad = outer.scratch().map(_.contains(s1_paddr)).getOrElse(Bool(false))
+      val baseAddr = GetPropertyByHartId(p(coreplex.RocketTilesKey), _.dcache.flatMap(_.scratch.map(_.U)), io.hartid)
+      val inScratchpad = s1_paddr >= baseAddr && s1_paddr < baseAddr + nSets * cacheBlockBytes
       val hitState = Mux(inScratchpad, ClientMetadata.maximum, ClientMetadata.onReset)
       (inScratchpad, hitState, L1Metadata(UInt(0), ClientMetadata.onReset))
     } else {
@@ -162,6 +163,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
   val s2_req = Reg(io.cpu.req.bits)
   val s2_req_block_addr = (s2_req.addr >> idxLSB) << idxLSB
   val s2_uncached = Reg(Bool())
+  val s2_uncached_resp_addr = Reg(UInt()) // should be DCE'd in synthesis
   when (s1_valid_not_nacked || s1_flush_valid) {
     s2_req := s1_req
     s2_req.addr := s1_paddr
@@ -358,6 +360,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
         s2_req.typ := req.typ
         s2_req.tag := req.tag
         s2_req.addr := Cat(s1_paddr >> beatOffBits /* don't-care */, req.addr(beatOffBits-1, 0))
+        s2_uncached_resp_addr := req.addr
       }
     } .elsewhen (grantIsVoluntary) {
       assert(release_ack_wait, "A ReleaseAck was unexpected by the dcache.") // TODO should handle Ack coming back on same cycle!
@@ -505,6 +508,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) {
     assert(!s2_valid_hit)
     io.cpu.resp.valid := true
     io.cpu.resp.bits.replay := true
+    io.cpu.resp.bits.addr := s2_uncached_resp_addr
   }
 
   // load data subword mux/sign extension

src/main/scala/rocket/Frontend.scala

@@ -42,34 +42,37 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) {
   val acquire = Bool(INPUT)
 }
 
-class Frontend(implicit p: Parameters) extends LazyModule {
+class Frontend(hartid: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new FrontendModule(this)
-  val icache = LazyModule(new ICache(latency = 2))
+  val icache = LazyModule(new ICache(latency = 2, hartid))
-  val node = TLOutputNode()
+  val masterNode = TLOutputNode()
+  val slaveNode = TLInputNode()
 
-  node := icache.node
+  icache.slaveNode.map { _ := slaveNode }
+  masterNode := icache.masterNode
 }
 
 class FrontendBundle(outer: Frontend) extends CoreBundle()(outer.p) {
   val cpu = new FrontendIO().flip
   val ptw = new TLBPTWIO()
-  val mem = outer.node.bundleOut
+  val tl_out = outer.masterNode.bundleOut
+  val tl_in = outer.slaveNode.bundleIn
   val resetVector = UInt(INPUT, vaddrBitsExtended)
+  val hartid = UInt(INPUT, hartIdLen)
 }
 
 class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
     with HasCoreParameters
     with HasL1ICacheParameters {
   val io = new FrontendBundle(outer)
-  implicit val edge = outer.node.edgesOut(0)
+  implicit val edge = outer.masterNode.edgesOut.head
   val icache = outer.icache.module
 
   val tlb = Module(new TLB(log2Ceil(coreInstBytes*fetchWidth), nTLBEntries))
   val fq = withReset(reset || io.cpu.req.valid) { Module(new ShiftQueue(new FrontendResp, 3, flow = true)) }
 
   val s0_valid = io.cpu.req.valid || fq.io.enq.ready
-  val s1_pc_ = Reg(UInt(width=vaddrBitsExtended))
+  val s1_pc = Reg(UInt(width=vaddrBitsExtended))
-  val s1_pc = ~(~s1_pc_ | (coreInstBytes-1)) // discard PC LSBS (this propagates down the pipeline)
   val s1_speculative = Reg(Bool())
   val s2_valid = Reg(init=Bool(true))
   val s2_pc = Reg(init=io.resetVector)
@@ -94,7 +97,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   s2_replay := (s2_valid && !fq.io.enq.fire()) || RegNext(s2_replay && !s0_valid)
   val npc = Mux(s2_replay, s2_pc, predicted_npc)
 
-  s1_pc_ := io.cpu.npc
+  s1_pc := io.cpu.npc
   // consider RVC fetches across blocks to be non-speculative if the first
   // part was non-speculative
   val s0_speculative =
@@ -116,7 +119,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   if (usingBTB) {
     val btb = Module(new BTB)
     btb.io.req.valid := false
-    btb.io.req.bits.addr := s1_pc_
+    btb.io.req.bits.addr := s1_pc
     btb.io.btb_update := io.cpu.btb_update
     btb.io.bht_update := io.cpu.bht_update
     btb.io.ras_update := io.cpu.ras_update
@@ -148,16 +151,18 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   tlb.io.req.bits.sfence := io.cpu.sfence
   tlb.io.req.bits.size := log2Ceil(coreInstBytes*fetchWidth)
 
+  icache.io.hartid := io.hartid
   icache.io.req.valid := s0_valid
   icache.io.req.bits.addr := io.cpu.npc
   icache.io.invalidate := io.cpu.flush_icache
   icache.io.s1_paddr := tlb.io.resp.paddr
+  icache.io.s2_vaddr := s2_pc
   icache.io.s1_kill := io.cpu.req.valid || tlb.io.resp.miss || s2_replay
   icache.io.s2_kill := s2_speculative && !s2_cacheable || s2_xcpt
 
   fq.io.enq.valid := s2_valid && (icache.io.resp.valid || icache.io.s2_kill)
   fq.io.enq.bits.pc := s2_pc
-  io.cpu.npc := Mux(io.cpu.req.valid, io.cpu.req.bits.pc, npc)
+  io.cpu.npc := ~(~Mux(io.cpu.req.valid, io.cpu.req.bits.pc, npc) | (coreInstBytes-1)) // discard LSB(s)
 
   fq.io.enq.bits.data := icache.io.resp.bits
   fq.io.enq.bits.mask := UInt((1 << fetchWidth)-1) << s2_pc.extract(log2Ceil(fetchWidth)+log2Ceil(coreInstBytes)-1, log2Ceil(coreInstBytes))
@@ -170,14 +175,15 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   io.cpu.resp <> fq.io.deq
 
   // performance events
-  io.cpu.acquire := edge.done(icache.io.mem(0).a)
+  io.cpu.acquire := edge.done(icache.io.tl_out(0).a)
 }
 
 /** Mix-ins for constructing tiles that have an ICache-based pipeline frontend */
 trait HasICacheFrontend extends CanHavePTW with HasTileLinkMasterPort {
   val module: HasICacheFrontendModule
-  val frontend = LazyModule(new Frontend)
+  val frontend = LazyModule(new Frontend(hartid: Int))
-  masterNode := frontend.node
+  val hartid: Int
+  masterNode := frontend.masterNode
   nPTWPorts += 1
 }
 

src/main/scala/rocket/HellaCache.scala

@@ -158,8 +158,8 @@ abstract class HellaCache(implicit p: Parameters) extends LazyModule {
   val module: HellaCacheModule
 }
 
-class HellaCacheBundle(outer: HellaCache) extends Bundle {
+class HellaCacheBundle(outer: HellaCache)(implicit p: Parameters) extends CoreBundle()(p) {
-  implicit val p = outer.p
+  val hartid = UInt(INPUT, hartIdLen)
   val cpu = (new HellaCacheIO).flip
   val ptw = new TLBPTWIO()
   val mem = outer.node.bundleOut
@@ -184,7 +184,7 @@ object HellaCache {
 
 /** Mix-ins for constructing tiles that have a HellaCache */
 
-trait HasHellaCache extends HasTileLinkMasterPort {
+trait HasHellaCache extends HasTileLinkMasterPort with HasTileParameters {
   val module: HasHellaCacheModule
   implicit val p: Parameters
   def findScratchpadFromICache: Option[AddressSet]

src/main/scala/rocket/ICache.scala

@@ -19,6 +19,7 @@ case class ICacheParams(
     nTLBEntries: Int = 32,
     cacheIdBits: Int = 0,
     ecc: Option[Code] = None,
+    itimAddr: Option[BigInt] = None,
     blockBytes: Int = 64) extends L1CacheParams {
   def replacement = new RandomReplacement(nWays)
 }
@@ -31,65 +32,114 @@ class ICacheReq(implicit p: Parameters) extends CoreBundle()(p) with HasL1ICache
   val addr = UInt(width = vaddrBits)
 }
 
-class ICacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasL1ICacheParameters {
+class ICache(val latency: Int, val hartid: Int)(implicit p: Parameters) extends LazyModule
-  val data = Bits(width = coreInstBits)
+    with HasRocketCoreParameters {
-  val datablock = Bits(width = rowBits)
-}
-
-class ICache(val latency: Int)(implicit p: Parameters) extends LazyModule {
   lazy val module = new ICacheModule(this)
-  val node = TLClientNode(TLClientParameters(sourceId = IdRange(0,1)))
+  val masterNode = TLClientNode(TLClientParameters(sourceId = IdRange(0,1)))
+
+  val icacheParams = tileParams.icache.get
+  val size = icacheParams.nSets * icacheParams.nWays * icacheParams.blockBytes
+  val slaveNode = icacheParams.itimAddr.map { itimAddr =>
+    val wordBytes = coreInstBytes * fetchWidth
+    TLManagerNode(Seq(TLManagerPortParameters(
+      Seq(TLManagerParameters(
+        address         = Seq(AddressSet(itimAddr, size-1)),
+        regionType      = RegionType.UNCACHED,
+        executable      = true,
+        supportsPutFull = TransferSizes(1, wordBytes),
+        supportsGet     = TransferSizes(1, wordBytes),
+        fifoId          = Some(0))), // requests handled in FIFO order
+      beatBytes = wordBytes,
+      minLatency = 1)))
+  }
 }
 
 class ICacheBundle(outer: ICache) extends CoreBundle()(outer.p) {
-  val req = Valid(new ICacheReq).flip
+  val hartid = UInt(INPUT, hartIdLen)
+  val req = Decoupled(new ICacheReq).flip
   val s1_paddr = UInt(INPUT, paddrBits) // delayed one cycle w.r.t. req
+  val s2_vaddr = UInt(INPUT, vaddrBits) // delayed two cycles w.r.t. req
   val s1_kill = Bool(INPUT) // delayed one cycle w.r.t. req
   val s2_kill = Bool(INPUT) // delayed two cycles; prevents I$ miss emission
 
   val resp = Valid(UInt(width = coreInstBits * fetchWidth))
   val invalidate = Bool(INPUT)
-  val mem = outer.node.bundleOut
+  val tl_out = outer.masterNode.bundleOut
+  val tl_in = outer.slaveNode.map(_.bundleIn)
+}
+
+// get a tile-specific property without breaking deduplication
+object GetPropertyByHartId {
+  def apply[T <: Data](tiles: Seq[RocketTileParams], f: RocketTileParams => Option[T], hartId: UInt): T = {
+    PriorityMux(tiles.zipWithIndex.collect { case (t, i) if f(t).nonEmpty => (hartId === i) -> f(t).get })
+  }
 }
 
 class ICacheModule(outer: ICache) extends LazyModuleImp(outer)
     with HasL1ICacheParameters {
   val io = new ICacheBundle(outer)
-  val edge = outer.node.edgesOut(0)
+  val edge_out = outer.masterNode.edgesOut.head
-  val tl_out = io.mem(0)
+  val tl_out = io.tl_out.head
+  val edge_in = outer.slaveNode.map(_.edgesIn.head)
+  val tl_in = io.tl_in.map(_.head)
 
   require(isPow2(nSets) && isPow2(nWays))
   require(isPow2(coreInstBytes))
   require(!usingVM || pgIdxBits >= untagBits)
 
+  val scratchpadOn = RegInit(false.B)
+  val scratchpadMax = tl_in.map(tl => Reg(UInt(width = log2Ceil(nSets * (nWays - 1)))))
+  def lineInScratchpad(line: UInt) = scratchpadMax.map(scratchpadOn && line <= _).getOrElse(false.B)
+  def addrMaybeInScratchpad(addr: UInt) = if (outer.icacheParams.itimAddr.isEmpty) false.B else {
+    val base = GetPropertyByHartId(p(coreplex.RocketTilesKey), _.icache.flatMap(_.itimAddr.map(_.U)), io.hartid)
+    addr >= base && addr < base + outer.size
+  }
+  def addrInScratchpad(addr: UInt) = addrMaybeInScratchpad(addr) && lineInScratchpad(addr(untagBits+log2Ceil(nWays)-1, blockOffBits))
+  def scratchpadWay(addr: UInt) = addr.extract(untagBits+log2Ceil(nWays)-1, untagBits)
+  def scratchpadWayValid(way: UInt) = way < nWays - 1
+  def scratchpadLine(addr: UInt) = addr(untagBits+log2Ceil(nWays)-1, blockOffBits)
+  val s0_slaveValid = tl_in.map(_.a.fire()).getOrElse(false.B)
+  val s1_slaveValid = RegNext(s0_slaveValid, false.B)
+  val s2_slaveValid = RegNext(s1_slaveValid, false.B)
+  val s3_slaveValid = RegNext(s2_slaveValid, false.B)
+
   val s_ready :: s_request :: s_refill :: Nil = Enum(UInt(), 3)
   val state = Reg(init=s_ready)
   val invalidated = Reg(Bool())
 
   val refill_addr = Reg(UInt(width = paddrBits))
+  val refill_tag = refill_addr(tagBits+untagBits-1,untagBits)
+  val refill_idx = refill_addr(untagBits-1,blockOffBits)
   val s1_tag_hit = Wire(Vec(nWays, Bool()))
-  val s1_any_tag_hit = s1_tag_hit.reduce(_||_)
+  val s1_any_tag_hit = s1_tag_hit.reduce(_||_) || Mux(s1_slaveValid, true.B, addrMaybeInScratchpad(io.s1_paddr))
 
   val s1_valid = Reg(init=Bool(false))
-  val out_valid = s1_valid && state === s_ready && !io.s1_kill
+  val s1_hit = s1_valid && s1_any_tag_hit
-  val s1_idx = io.s1_paddr(untagBits-1,blockOffBits)
-  val s1_tag = io.s1_paddr(tagBits+untagBits-1,untagBits)
-  val s1_hit = out_valid && s1_any_tag_hit
   val s1_miss = s1_valid && state === s_ready && !s1_any_tag_hit
 
-  val s0_valid = io.req.valid && state === s_ready
+  io.req.ready := !(tl_out.d.fire() || s0_slaveValid || s3_slaveValid)
+  val s0_valid = io.req.fire()
   val s0_vaddr = io.req.bits.addr
 
   s1_valid := s0_valid
 
   when (s1_miss) { refill_addr := io.s1_paddr }
-  val refill_tag = refill_addr(tagBits+untagBits-1,untagBits)
+  val (_, _, refill_done, refill_cnt) = edge_out.count(tl_out.d)
-  val refill_idx = refill_addr(untagBits-1,blockOffBits)
+  tl_out.d.ready := !s3_slaveValid
-  val (_, _, refill_done, refill_cnt) = edge.count(tl_out.d)
+  require (edge_out.manager.minLatency > 0)
-  tl_out.d.ready := Bool(true)
+
-  require (edge.manager.minLatency > 0)
+  val repl_way = if (isDM) UInt(0) else {
+    // pick a way that is not used by the scratchpad
+    val v0 = LFSR16(tl_out.a.fire())(log2Up(nWays)-1,0)
+    var v = v0
+    for (i <- log2Ceil(nWays) - 1 to 0 by -1) {
+      val mask = nWays - (BigInt(1) << (i + 1))
+      v = v | (lineInScratchpad(Cat(v0 | mask.U, refill_idx)) << i)
+    }
+    assert(!lineInScratchpad(Cat(v, refill_idx)))
+    v
+  }
 
-  val repl_way = if (isDM) UInt(0) else LFSR16(s1_miss)(log2Up(nWays)-1,0)
   val entagbits = code.width(tagBits)
   val tag_array = SeqMem(nSets, Vec(nWays, Bits(width = entagbits)))
   val tag_rdata = tag_array.read(s0_vaddr(untagBits-1,blockOffBits), !refill_done && s0_valid)
@@ -99,8 +149,9 @@ class ICacheModule(outer: ICache) extends LazyModuleImp(outer)
   }
 
   val vb_array = Reg(init=Bits(0, nSets*nWays))
-  when (refill_done && !invalidated) {
+  when (tl_out.d.fire()) {
-    vb_array := vb_array.bitSet(Cat(repl_way, refill_idx), Bool(true))
+    // clear bit when refill starts so hit-under-miss doesn't fetch bad data
+    vb_array := vb_array.bitSet(Cat(repl_way, refill_idx), refill_done && !invalidated)
   }
   val invalidate = Wire(init = io.invalidate)
   when (invalidate) {
@@ -111,72 +162,127 @@ class ICacheModule(outer: ICache) extends LazyModuleImp(outer)
   val s1_tag_disparity = Wire(Vec(nWays, Bool()))
   val wordBits = coreInstBits * fetchWidth
   val s1_dout = Wire(Vec(nWays, UInt(width = code.width(wordBits))))
-  val s1_dout_valid = RegNext(s0_valid)
+
+  val s0_slaveAddr = tl_in.map(_.a.bits.address).getOrElse(0.U)
+  val s1s3_slaveAddr = Reg(UInt(width = log2Ceil(outer.size)))
+  val s1s3_slaveData = Reg(UInt(width = wordBits))
 
   for (i <- 0 until nWays) {
-    val s1_vb = vb_array(Cat(UInt(i), io.s1_paddr(untagBits-1,blockOffBits))).toBool
+    val s1_idx = io.s1_paddr(untagBits-1,blockOffBits)
-    s1_tag_disparity(i) := code.decode(tag_rdata(i)).error
+    val s1_tag = io.s1_paddr(tagBits+untagBits-1,untagBits)
-    s1_tag_hit(i) := s1_vb && code.decode(tag_rdata(i)).uncorrected === s1_tag
+    val scratchpadHit = scratchpadWayValid(i) &&
+      Mux(s1_slaveValid,
+        lineInScratchpad(scratchpadLine(s1s3_slaveAddr)) && scratchpadWay(s1s3_slaveAddr) === i,
+        addrInScratchpad(io.s1_paddr) && scratchpadWay(io.s1_paddr) === i)
+    val s1_vb = vb_array(Cat(UInt(i), s1_idx)) && !s1_slaveValid
+    s1_tag_disparity(i) := s1_vb && code.decode(tag_rdata(i)).error
+    s1_tag_hit(i) := scratchpadHit || (s1_vb && code.decode(tag_rdata(i)).uncorrected === s1_tag)
   }
+  assert(!(s1_valid || s1_slaveValid) || PopCount(s1_tag_hit zip s1_tag_disparity map { case (h, d) => h && !d }) <= 1)
 
-  require(rowBits % wordBits == 0)
+  require(tl_out.d.bits.data.getWidth % wordBits == 0)
-  val data_arrays = Seq.fill(rowBits / wordBits) { SeqMem(nSets * refillCycles, Vec(nWays, UInt(width = code.width(wordBits)))) }
+  val data_arrays = Seq.fill(tl_out.d.bits.data.getWidth / wordBits) { SeqMem(nSets * refillCycles, Vec(nWays, UInt(width = code.width(wordBits)))) }
   for ((data_array, i) <- data_arrays zipWithIndex) {
-    val wen = tl_out.d.valid
+    def wordMatch(addr: UInt) = addr.extract(log2Ceil(tl_out.d.bits.data.getWidth/8)-1, log2Ceil(wordBits/8)) === i
+    def row(addr: UInt) = addr(untagBits-1, blockOffBits-log2Ceil(refillCycles))
+    val s0_ren = (s0_valid && wordMatch(s0_vaddr)) || (s0_slaveValid && wordMatch(s0_slaveAddr))
+    val wen = (tl_out.d.fire() && !invalidated) || (s3_slaveValid && wordMatch(s1s3_slaveAddr))
+    val mem_idx = Mux(tl_out.d.fire(), (refill_idx << log2Ceil(refillCycles)) | refill_cnt,
+                  Mux(s3_slaveValid, row(s1s3_slaveAddr),
+                  Mux(s0_slaveValid, row(s0_slaveAddr),
+                  row(s0_vaddr))))
     when (wen) {
-      val idx = (refill_idx << log2Ceil(refillCycles)) | refill_cnt
+      val data = Mux(s3_slaveValid, s1s3_slaveData, tl_out.d.bits.data(wordBits*(i+1)-1, wordBits*i))
-      val data = tl_out.d.bits.data(wordBits*(i+1)-1, wordBits*i)
+      val way = Mux(s3_slaveValid, scratchpadWay(s1s3_slaveAddr), repl_way)
-      data_array.write(idx, Vec.fill(nWays)(code.encode(data)), (0 until nWays).map(repl_way === _))
+      data_array.write(mem_idx, Vec.fill(nWays)(code.encode(data)), (0 until nWays).map(way === _))
     }
-    def wordMatch(addr: UInt) = addr.extract(log2Ceil(rowBytes)-1, log2Ceil(wordBits/8)) === i
+    val dout = data_array.read(mem_idx, !wen && s0_ren)
-    val s0_raddr = s0_vaddr(untagBits-1,blockOffBits-log2Ceil(refillCycles))
+    when (wordMatch(Mux(s1_slaveValid, s1s3_slaveAddr, io.s1_paddr))) {
-    val dout = data_array.read(s0_raddr, !wen && (s0_valid && wordMatch(s0_vaddr)))
-    when (wordMatch(io.s1_paddr)) {
       s1_dout := dout
     }
   }
 
-/*
-  for ((data_array, i) <- data_arrays zipWithIndex) {
-    val wen = tl_out.d.valid && repl_way === UInt(i)
-    when (wen) {
-      val e_d = code.encode(tl_out.d.bits.data)
-      data_array.write((refill_idx << log2Ceil(refillCycles)) | refill_cnt, e_d)
-    }
-    val s0_raddr = s0_vaddr(untagBits-1,blockOffBits-log2Ceil(refillCycles))
-    s1_dout(i) := data_array.read(s0_raddr, !wen && s0_valid)
-  }
-*/
-
   // output signals
   outer.latency match {
     case 1 =>
-      require(code.width(rowBits) == rowBits) // no ECC
+      require(code.isInstanceOf[uncore.util.IdentityCode])
+      require(outer.icacheParams.itimAddr.isEmpty)
       io.resp.bits := Mux1H(s1_tag_hit, s1_dout)
       io.resp.valid := s1_hit
+
     case 2 =>
-      val s2_valid = RegNext(out_valid, Bool(false))
+      val s2_valid = RegNext(s1_valid && !io.s1_kill, Bool(false))
       val s2_hit = RegNext(s1_hit, Bool(false))
-      val s2_tag_hit = RegEnable(s1_tag_hit, s1_valid)
+      val s2_tag_hit = RegEnable(s1_tag_hit, s1_valid || s1_slaveValid)
-      val s2_dout = RegEnable(s1_dout, s1_valid)
+      val s2_dout = RegEnable(s1_dout, s1_valid || s1_slaveValid)
       val s2_way_mux = Mux1H(s2_tag_hit, s2_dout)
 
-      val s2_tag_disparity = RegEnable(s1_tag_disparity, s1_valid).asUInt.orR
+      val s2_tag_disparity = RegEnable(s1_tag_disparity, s1_valid || s1_slaveValid).asUInt.orR
-      val s2_data_disparity = code.decode(s2_way_mux).error
+      val s2_data_decoded = code.decode(s2_way_mux)
-      val s2_disparity = s2_tag_disparity || s2_data_disparity
+      val s2_disparity = s2_tag_disparity || s2_data_decoded.error
       when (s2_valid && s2_disparity) { invalidate := true }
 
-      io.resp.bits := code.decode(s2_way_mux).uncorrected
+      io.resp.bits := s2_data_decoded.uncorrected
       io.resp.valid := s2_hit && !s2_disparity
+
+      tl_in.map { tl =>
+        tl.a.ready := !tl_out.d.fire() && !s1_slaveValid && !s2_slaveValid && !(tl.d.valid && !tl.d.ready)
+        val s1_a = RegEnable(tl.a.bits, s0_slaveValid)
+        when (s0_slaveValid) {
+          val a = tl.a.bits
+          s1s3_slaveAddr := tl.a.bits.address
+          s1s3_slaveData := tl.a.bits.data
+          when (edge_in.get.hasData(a)) {
+            val enable = scratchpadWayValid(scratchpadWay(a.address))
+            when (!lineInScratchpad(scratchpadLine(a.address))) {
+              scratchpadMax.get := scratchpadLine(a.address)
+              when (enable) { invalidate := true }
+            }
+            scratchpadOn := enable
+          }
+        }
+
+        assert(!s2_valid || RegNext(RegNext(s0_vaddr)) === io.s2_vaddr)
+        when (!(tl.a.valid || s1_slaveValid || s2_slaveValid)
+              && s2_valid && s2_data_decoded.correctable && !s2_tag_disparity) {
+          // handle correctable errors on CPU accesses to the scratchpad.
+          // if there is an in-flight slave-port access to the scratchpad,
+          // report the a miss but don't correct the error (as there is
+          // a structural hazard on s1s3_slaveData/s1s3_slaveAddress).
+          s3_slaveValid := true
+          s1s3_slaveData := s2_data_decoded.corrected
+          s1s3_slaveAddr := Cat(OHToUInt(s2_tag_hit), io.s2_vaddr(untagBits-1, log2Ceil(wordBits/8)), s1s3_slaveAddr(log2Ceil(wordBits/8)-1, 0))
+        }
+
+        val respValid = RegInit(false.B)
+        respValid := s2_slaveValid || (respValid && !tl.d.ready)
+        when (s2_slaveValid) {
+          when (edge_in.get.hasData(s1_a)) { s3_slaveValid := true }
+          def byteEn(i: Int) = !(edge_in.get.hasData(s1_a) && s1_a.mask(i))
+          s1s3_slaveData := (0 until wordBits/8).map(i => Mux(byteEn(i), s2_data_decoded.corrected, s1s3_slaveData)(8*(i+1)-1, 8*i)).asUInt
+        }
+
+        tl.d.valid := respValid
+        tl.d.bits := Mux(edge_in.get.hasData(s1_a),
+          edge_in.get.AccessAck(s1_a, UInt(0)),
+          edge_in.get.AccessAck(s1_a, UInt(0), UInt(0)))
+        tl.d.bits.data := s1s3_slaveData
+
+        // Tie off unused channels
+        tl.b.valid := false
+        tl.c.ready := true
+        tl.e.ready := true
+      }
   }
   tl_out.a.valid := state === s_request && !io.s2_kill
-  tl_out.a.bits := edge.Get(
+  tl_out.a.bits := edge_out.Get(
                     fromSource = UInt(0),
                     toAddress = (refill_addr >> blockOffBits) << blockOffBits,
                     lgSize = lgCacheBlockBytes)._2
   tl_out.b.ready := Bool(true)
   tl_out.c.valid := Bool(false)
   tl_out.e.valid := Bool(false)
+  assert(!(tl_out.a.valid && addrMaybeInScratchpad(tl_out.a.bits.address)))
 
   // control state machine
   switch (state) {

src/main/scala/rocket/PMP.scala

@@ -42,7 +42,7 @@ class PMP(implicit p: Parameters) extends PMPReg {
   val mask = UInt(width = paddrBits)
 
   import PMP._
-  def computeMask = Cat((Cat(addr, cfg.a(0)) + 1) ^ Cat(addr, cfg.a(0)), UInt((BigInt(1) << (lgAlign-1)) - 1, lgAlign-1))
+  def computeMask = Cat(Cat(addr, cfg.a(0)) & ~(Cat(addr, cfg.a(0)) + 1), UInt((BigInt(1) << lgAlign) - 1, lgAlign))
   private def comparand = addr << lgAlign
 
   private def pow2Match(x: UInt, lgSize: UInt, lgMaxSize: Int) = {

src/main/scala/rocket/Rocket.scala

@@ -8,8 +8,10 @@ import chisel3.core.withReset
 import config._
 import tile._
 import uncore.constants._
+import diplomacy._
 import util._
 import Chisel.ImplicitConversions._
+import collection.immutable.ListMap
 
 case class RocketCoreParams(
   bootFreqHz: BigInt = 0,

src/main/scala/rocket/ScratchpadSlavePort.scala

@@ -105,17 +105,25 @@ class ScratchpadSlavePort(address: AddressSet)(implicit p: Parameters) extends L
 }
 
 /** Mix-ins for constructing tiles that have optional scratchpads */
-trait CanHaveScratchpad extends HasHellaCache with HasICacheFrontend {
+trait CanHaveScratchpad extends HasHellaCache with HasICacheFrontend with HasCoreParameters {
   val module: CanHaveScratchpadModule
 
+  val slaveNode = TLInputNode() // Up to two uses for this input node:
+
+  // 1) Frontend always exists, but may or may not have a scratchpad node
+  val fg = LazyModule(new TLFragmenter(fetchWidth*coreInstBytes, p(CacheBlockBytes), true))
+  val ww = LazyModule(new TLWidthWidget(xLen/8))
+  frontend.slaveNode :*= fg.node
+  fg.node :*= ww.node
+  ww.node :*= slaveNode
+
+  // 2) ScratchpadSlavePort always has a node, but only exists when the HellaCache has a scratchpad
   val scratch = tileParams.dcache.flatMap(d => d.scratch.map(s =>
     LazyModule(new ScratchpadSlavePort(AddressSet(s, d.dataScratchpadBytes-1)))))
-  val slaveNode = TLInputNode()
+  scratch foreach { lm => lm.node := TLFragmenter(xLen/8, p(CacheBlockBytes))(slaveNode) }
-
-  scratch foreach { lm => lm.node := TLFragmenter(p(XLen)/8, p(CacheBlockBytes))(slaveNode) }
 
   def findScratchpadFromICache: Option[AddressSet] = scratch.map { s =>
-    val finalNode = frontend.node.edgesOut(0).manager.managers.find(_.nodePath.last == s.node)
+    val finalNode = frontend.masterNode.edgesOut.head.manager.managers.find(_.nodePath.last == s.node)
     require (finalNode.isDefined, "Could not find the scratch pad; not reachable via icache?")
     require (finalNode.get.address.size == 1, "Scratchpad address space was fragmented!")
     finalNode.get.address(0)

src/main/scala/rocket/Tile.scala

@@ -132,6 +132,8 @@ class RocketTileModule(outer: RocketTile) extends BaseTileModule(outer, () => ne
   core.io.hartid := io.hartid // Pass through the hartid
   outer.frontend.module.io.cpu <> core.io.imem
   outer.frontend.module.io.resetVector := io.resetVector
+  outer.frontend.module.io.hartid := io.hartid
+  outer.dcache.module.io.hartid := io.hartid
   dcachePorts += core.io.dmem // TODO outer.dcachePorts += () => module.core.io.dmem ??
   fpuOpt foreach { fpu => core.io.fpu <> fpu.io }
   core.io.ptw <> ptw.io.dpath
@@ -174,12 +176,10 @@ class SyncRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Parameters)
   xing.intnode := intNode
 
   lazy val module = new LazyModuleImp(this) {
-    val io = new Bundle {
+    val io = new CoreBundle with HasExternallyDrivenTileConstants {
       val master = masterNode.bundleOut
       val slave = slaveNode.bundleIn
       val interrupts = intNode.bundleIn
-      val hartid = UInt(INPUT, p(XLen))
-      val resetVector = UInt(INPUT, p(XLen))
     }
     // signals that do not change:
     rocket.module.io.hartid := io.hartid
@@ -206,12 +206,10 @@ class AsyncRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Parameters
   xing.intnode := intNode
 
   lazy val module = new LazyModuleImp(this) {
-    val io = new Bundle {
+    val io = new CoreBundle with HasExternallyDrivenTileConstants {
       val master = masterNode.bundleOut
       val slave = slaveNode.bundleIn
       val interrupts = intNode.bundleIn
-      val hartid = UInt(INPUT, p(XLen))
-      val resetVector = UInt(INPUT, p(XLen))
     }
     // signals that do not change:
     rocket.module.io.hartid := io.hartid
@@ -242,12 +240,10 @@ class RationalRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Paramet
   xing.intnode := intNode
 
   lazy val module = new LazyModuleImp(this) {
-    val io = new Bundle {
+    val io = new CoreBundle with HasExternallyDrivenTileConstants {
       val master = masterNode.bundleOut
       val slave = slaveNode.bundleIn
       val interrupts = intNode.bundleIn
-      val hartid = UInt(INPUT, p(XLen))
-      val resetVector = UInt(INPUT, p(XLen))
     }
     // signals that do not change:
     rocket.module.io.hartid := io.hartid

src/main/scala/rocketchip/RocketPlexMaster.scala

@@ -3,17 +3,13 @@
 package rocketchip
 
 import Chisel._
-import config._
+import coreplex.RocketPlex
-import diplomacy._
+import diplomacy.LazyModule
-import uncore.tilelink2._
-import uncore.devices._
-import util._
-import coreplex._
 
 trait RocketPlexMaster extends HasTopLevelNetworks {
   val module: RocketPlexMasterModule
 
-  val coreplex = LazyModule(new DefaultCoreplex)
+  val coreplex = LazyModule(new RocketPlex)
 
   coreplex.l2in :=* fsb.node
   bsb.node :*= coreplex.l2out

src/main/scala/tile/BaseTile.scala

@@ -11,6 +11,8 @@ import util._
 
 case object SharedMemoryTLEdge extends Field[TLEdgeOut]
 case object TileKey extends Field[TileParams]
+case object ResetVectorBits extends Field[Int]
+case object MaxHartIdBits extends Field[Int]
 
 trait TileParams {
   val core: CoreParams
@@ -31,6 +33,7 @@ trait HasTileParameters {
   val usingBTB = tileParams.btb.isDefined && tileParams.btb.get.nEntries > 0
   val usingPTW = usingVM
   val usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined
+  val hartIdLen = p(MaxHartIdBits)
 
   def dcacheArbPorts = 1 + usingVM.toInt + usingDataScratchpad.toInt + tileParams.rocc.size
 }
@@ -47,8 +50,8 @@ abstract class BareTileModule[+L <: BareTile, +B <: BareTileBundle[L]](_outer: L
   val io = _io ()
 }
 
-// Uses TileLink master port to connect caches and accelerators to the coreplex
+/** Uses TileLink master port to connect caches and accelerators to the coreplex */
-trait HasTileLinkMasterPort extends HasTileParameters {
+trait HasTileLinkMasterPort {
   implicit val p: Parameters
   val module: HasTileLinkMasterPortModule
   val masterNode = TLOutputNode()
@@ -64,6 +67,14 @@ trait HasTileLinkMasterPortModule {
   val io: HasTileLinkMasterPortBundle
 }
 
+/** Some other standard inputs */
+trait HasExternallyDrivenTileConstants extends Bundle {
+  implicit val p: Parameters
+  val hartid = UInt(INPUT, p(MaxHartIdBits))
+  val resetVector = UInt(INPUT, p(ResetVectorBits))
+}
+
+/** Base class for all Tiles that use TileLink */
 abstract class BaseTile(tileParams: TileParams)(implicit p: Parameters) extends BareTile
     with HasTileLinkMasterPort
     with HasExternalInterrupts {
@@ -71,11 +82,10 @@ abstract class BaseTile(tileParams: TileParams)(implicit p: Parameters) extends
 }
 
 class BaseTileBundle[+L <: BaseTile](_outer: L) extends BareTileBundle(_outer)
+    with HasTileParameters
     with HasTileLinkMasterPortBundle
-    with HasExternalInterruptsBundle {
+    with HasExternalInterruptsBundle
-  val hartid = UInt(INPUT, p(XLen))
+    with HasExternallyDrivenTileConstants
-  val resetVector = UInt(INPUT, p(XLen))
-}
 
 class BaseTileModule[+L <: BaseTile, +B <: BaseTileBundle[L]](_outer: L, _io: () => B) extends BareTileModule(_outer, _io)
     with HasTileLinkMasterPortModule

src/main/scala/tile/Core.scala

@@ -75,13 +75,12 @@ abstract class CoreModule(implicit val p: Parameters) extends Module
 abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p)
   with HasCoreParameters
 
-trait HasCoreIO {
+trait HasCoreIO extends HasTileParameters {
   implicit val p: Parameters
-  val io = new Bundle {
+  val io = new CoreBundle()(p) with HasExternallyDrivenTileConstants {
     val interrupts = new TileInterrupts().asInput
-    val hartid = UInt(INPUT, p(XLen))
+    val imem  = new FrontendIO
-    val imem  = new FrontendIO()(p)
+    val dmem = new HellaCacheIO
-    val dmem = new HellaCacheIO()(p)
     val ptw = new DatapathPTWIO().flip
     val fpu = new FPUCoreIO().flip
     val rocc = new RoCCCoreIO().flip

src/main/scala/uncore/devices/Plic.scala

@@ -12,6 +12,7 @@ import uncore.tilelink2._
 import config._
 import scala.math.min
 import tile.XLen
+import util._
 
 class GatewayPLICIO extends Bundle {
   val valid = Bool(OUTPUT)
@@ -76,7 +77,7 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame
     device    = device,
     beatBytes = p(XLen)/8,
     undefZero = false,
-    concurrency = 1) // Work around the enable -> claim hazard
+    concurrency = 1) // limiting concurrency handles RAW hazards on claim registers
 
   val intnode = IntNexusNode(
     numSourcePorts = 0 to 1024,
@@ -128,11 +129,11 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame
     require(nHarts > 0 && nHarts <= PLICConsts.maxHarts)
 
     // For now, use LevelGateways for all TL2 interrupts
-    val gateways = Vec(interrupts.map { case i =>
+    val gateways = Vec((false.B +: interrupts).map { case i =>
       val gateway = Module(new LevelGateway)
       gateway.io.interrupt := i
       gateway.io.plic
-    } ++ (if (interrupts.isEmpty) Some(Wire(new GatewayPLICIO)) else None))
+    })
 
     val priority =
       if (nPriorities > 0) Reg(Vec(nDevices+1, UInt(width=log2Up(nPriorities+1))))
@@ -143,7 +144,7 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame
     val pending = Reg(init=Vec.fill(nDevices+1){Bool(false)})
     val enables = Reg(Vec(nHarts, Vec(nDevices+1, Bool())))
 
-    for ((p, g) <- pending.tail zip gateways) {
+    for ((p, g) <- pending zip gateways) {
       g.ready := !p
       g.complete := false
       when (g.valid) { p := true }
@@ -152,19 +153,18 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame
     def findMax(x: Seq[UInt]): (UInt, UInt) = {
       if (x.length > 1) {
         val half = 1 << (log2Ceil(x.length) - 1)
-        val lMax = findMax(x take half)
+        val left = findMax(x take half)
-        val rMax = findMax(x drop half)
+        val right = findMax(x drop half)
-        val useLeft = lMax._1 >= rMax._1
+        MuxT(left._1 >= right._1, left, (right._1, UInt(half) | right._2))
-        (Mux(useLeft, lMax._1, rMax._1), Mux(useLeft, lMax._2, UInt(half) | rMax._2))
       } else (x.head, UInt(0))
     }
 
     val maxDevs = Reg(Vec(nHarts, UInt(width = log2Up(pending.size))))
     for (hart <- 0 until nHarts) {
-      val effectivePriority =
+      val effectivePriority = (UInt(1) << priority(0).getWidth) +:
-        for (((p, en), pri) <- (pending zip enables(hart) zip priority).tail)
+        (for (((p, en), pri) <- (pending zip enables(hart) zip priority).tail)
-          yield Cat(p && en, pri)
+          yield Cat(p && en, pri))
-      val (maxPri, maxDev) = findMax((UInt(1) << priority(0).getWidth) +: effectivePriority)
+      val (maxPri, maxDev) = findMax(effectivePriority)
 
       maxDevs(hart) := maxDev
       harts(hart) := Reg(next = maxPri) > Cat(UInt(1), threshold(hart))
@@ -185,13 +185,13 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame
           RegReadFn { valid =>
             when (valid) {
               pending(maxDevs(i)) := Bool(false)
-              maxDevs(i) := UInt(0) // flush pipeline
             }
             (Bool(true), maxDevs(i))
           },
           RegWriteFn { (valid, data) =>
-            when (valid && enables(i)(data)) {
+            val irq = data.extract(log2Ceil(nDevices+1)-1, 0)
-              gateways(data - UInt(1)).complete := Bool(true)
+            when (valid && enables(i)(irq)) {
+              gateways(irq).complete := Bool(true)
             }
             Bool(true)
           }

src/main/scala/uncore/util/Ecc.scala

@@ -38,10 +38,10 @@ class ParityCode extends Code
   def width(w0: Int) = w0+1
   def encode(x: UInt) = Cat(x.xorR, x)
   def decode(y: UInt) = new Decoding {
-    def uncorrected = y(y.getWidth-2,0)
+    val uncorrected = y(y.getWidth-2,0)
-    def corrected = uncorrected
+    val corrected = uncorrected
-    def correctable = Bool(false)
+    val correctable = Bool(false)
-    def uncorrectable = y.xorR
+    val uncorrectable = y.xorR
   }
 }
 
@@ -78,10 +78,10 @@ class SECCode extends Code
     }).asUInt
 
     private def swizzle(z: UInt) = (1 to n).filter(i => !isPow2(i)).map(i => z(i-1)).asUInt
-    def uncorrected = swizzle(y)
+    val uncorrected = swizzle(y)
-    def corrected = swizzle(((y << 1) ^ UIntToOH(syndrome)) >> 1)
+    val corrected = swizzle(((y << 1) ^ UIntToOH(syndrome)) >> 1)
-    def correctable = syndrome.orR
+    val correctable = syndrome.orR
-    def uncorrectable = syndrome > UInt(n)
+    val uncorrectable = syndrome > UInt(n)
   }
   private def mapping(i: Int) = i-1-log2Up(i)
 }
@@ -97,10 +97,10 @@ class SECDEDCode extends Code
     val secdec = sec.decode(x(x.getWidth-2,0))
     val pardec = par.decode(x)
 
-    def uncorrected = secdec.uncorrected
+    val uncorrected = secdec.uncorrected
-    def corrected = secdec.corrected
+    val corrected = secdec.corrected
-    def correctable = pardec.uncorrectable
+    val correctable = pardec.uncorrectable
-    def uncorrectable = !pardec.uncorrectable && secdec.correctable
+    val uncorrectable = !pardec.uncorrectable && secdec.correctable
   }
 }
 

src/main/scala/util/Package.scala

@@ -11,7 +11,9 @@ package object util {
 
   implicit class SeqToAugmentedSeq[T <: Data](val x: Seq[T]) extends AnyVal {
     def apply(idx: UInt): T = {
-      if (!isPow2(x.size)) {
+      if (x.size == 1) {
+        x.head
+      } else if (!isPow2(x.size)) {
         // For non-power-of-2 seqs, reflect elements to simplify decoder
         (x ++ x.takeRight(x.size & -x.size)).toSeq(idx)
       } else {