diff --git a/src/main/scala/coreplex/Configs.scala b/src/main/scala/coreplex/Configs.scala index a5ccc232..3a72cb94 100644 --- a/src/main/scala/coreplex/Configs.scala +++ b/src/main/scala/coreplex/Configs.scala @@ -19,6 +19,9 @@ class BaseCoreplexConfig extends Config ((site, here, up) => { case PgLevels => if (site(XLen) == 64) 3 /* Sv39 */ else 2 /* Sv32 */ case ASIdBits => 0 case XLen => 64 // Applies to all cores + case ResetVectorBits => site(PAddrBits) + case MaxHartIdBits => log2Up(site(NTiles)) + case MaxPriorityLevels => 7 case BuildCore => (p: Parameters) => new Rocket()(p) case RocketCrossing => SynchronousCrossing() case RocketTilesKey => Nil diff --git a/src/main/scala/coreplex/Coreplex.scala b/src/main/scala/coreplex/Coreplex.scala deleted file mode 100644 index 4e844c8d..00000000 --- a/src/main/scala/coreplex/Coreplex.scala +++ /dev/null @@ -1,24 +0,0 @@ -// See LICENSE.SiFive for license details. - -package coreplex - -import Chisel._ -import config._ -import diplomacy._ -import util._ - -///// - -class DefaultCoreplex(implicit p: Parameters) extends BaseCoreplex - with CoreplexRISCVPlatform - with HasRocketTiles { - override lazy val module = new DefaultCoreplexModule(this, () => new DefaultCoreplexBundle(this)) -} - -class DefaultCoreplexBundle[+L <: DefaultCoreplex](_outer: L) extends BaseCoreplexBundle(_outer) - with CoreplexRISCVPlatformBundle - with HasRocketTilesBundle - -class DefaultCoreplexModule[+L <: DefaultCoreplex, +B <: DefaultCoreplexBundle[L]](_outer: L, _io: () => B) extends BaseCoreplexModule(_outer, _io) - with CoreplexRISCVPlatformModule - with HasRocketTilesModule diff --git a/src/main/scala/coreplex/RISCVPlatform.scala b/src/main/scala/coreplex/RISCVPlatform.scala index 1a8850b8..ec65c192 100644 --- a/src/main/scala/coreplex/RISCVPlatform.scala +++ b/src/main/scala/coreplex/RISCVPlatform.scala @@ -3,19 +3,20 @@ package coreplex import Chisel._ -import config._ -import junctions._ +import config.Field import diplomacy._ import tile._ import uncore.tilelink2._ import uncore.devices._ import util._ +case object MaxPriorityLevels extends Field[Int] + trait CoreplexRISCVPlatform extends CoreplexNetwork { val module: CoreplexRISCVPlatformModule val debug = LazyModule(new TLDebugModule()) - val plic = LazyModule(new TLPLIC(maxPriorities = 7)) + val plic = LazyModule(new TLPLIC(maxPriorities = p(MaxPriorityLevels))) val clint = LazyModule(new CoreplexLocalInterrupter) debug.node := TLFragmenter(cbus_beatBytes, cbus_lineBytes)(cbus.node) @@ -34,7 +35,7 @@ trait CoreplexRISCVPlatformBundle extends CoreplexNetworkBundle { val debug = new ClockedDMIIO().flip val rtcToggle = Bool(INPUT) - val resetVector = UInt(INPUT, p(XLen)) + val resetVector = UInt(INPUT, p(ResetVectorBits)) val ndreset = Bool(OUTPUT) val dmactive = Bool(OUTPUT) } diff --git a/src/main/scala/coreplex/RocketPlex.scala b/src/main/scala/coreplex/RocketPlex.scala new file mode 100644 index 00000000..25c25ca1 --- /dev/null +++ b/src/main/scala/coreplex/RocketPlex.scala @@ -0,0 +1,20 @@ +// See LICENSE.SiFive for license details. + +package coreplex + +import Chisel._ +import config.Parameters + +class RocketPlex(implicit p: Parameters) extends BaseCoreplex + with CoreplexRISCVPlatform + with HasRocketTiles { + override lazy val module = new RocketPlexModule(this, () => new RocketPlexBundle(this)) +} + +class RocketPlexBundle[+L <: RocketPlex](_outer: L) extends BaseCoreplexBundle(_outer) + with CoreplexRISCVPlatformBundle + with HasRocketTilesBundle + +class RocketPlexModule[+L <: RocketPlex, +B <: RocketPlexBundle[L]](_outer: L, _io: () => B) extends BaseCoreplexModule(_outer, _io) + with CoreplexRISCVPlatformModule + with HasRocketTilesModule diff --git a/src/main/scala/rocket/CSR.scala b/src/main/scala/rocket/CSR.scala index 661cb1b8..9dc4240f 100644 --- a/src/main/scala/rocket/CSR.scala +++ b/src/main/scala/rocket/CSR.scala @@ -153,7 +153,7 @@ class PerfCounterIO(implicit p: Parameters) extends CoreBundle class CSRFileIO(implicit p: Parameters) extends CoreBundle with HasRocketCoreParameters { val interrupts = new TileInterrupts().asInput - val hartid = UInt(INPUT, xLen) + val hartid = UInt(INPUT, hartIdLen) val rw = new Bundle { val addr = UInt(INPUT, CSR.ADDRSZ) val cmd = Bits(INPUT, CSR.SZ) diff --git a/src/main/scala/rocket/DCache.scala b/src/main/scala/rocket/DCache.scala index 63affc8d..67163a6f 100644 --- a/src/main/scala/rocket/DCache.scala +++ b/src/main/scala/rocket/DCache.scala @@ -138,7 +138,8 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) { if (usingDataScratchpad) { metaWriteArb.io.out.ready := true metaReadArb.io.out.ready := !metaWriteArb.io.out.valid - val inScratchpad = outer.scratch().map(_.contains(s1_paddr)).getOrElse(Bool(false)) + val baseAddr = GetPropertyByHartId(p(coreplex.RocketTilesKey), _.dcache.flatMap(_.scratch.map(_.U)), io.hartid) + val inScratchpad = s1_paddr >= baseAddr && s1_paddr < baseAddr + nSets * cacheBlockBytes val hitState = Mux(inScratchpad, ClientMetadata.maximum, ClientMetadata.onReset) (inScratchpad, hitState, L1Metadata(UInt(0), ClientMetadata.onReset)) } else { @@ -162,6 +163,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) { val s2_req = Reg(io.cpu.req.bits) val s2_req_block_addr = (s2_req.addr >> idxLSB) << idxLSB val s2_uncached = Reg(Bool()) + val s2_uncached_resp_addr = Reg(UInt()) // should be DCE'd in synthesis when (s1_valid_not_nacked || s1_flush_valid) { s2_req := s1_req s2_req.addr := s1_paddr @@ -358,6 +360,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) { s2_req.typ := req.typ s2_req.tag := req.tag s2_req.addr := Cat(s1_paddr >> beatOffBits /* don't-care */, req.addr(beatOffBits-1, 0)) + s2_uncached_resp_addr := req.addr } } .elsewhen (grantIsVoluntary) { assert(release_ack_wait, "A ReleaseAck was unexpected by the dcache.") // TODO should handle Ack coming back on same cycle! @@ -505,6 +508,7 @@ class DCacheModule(outer: DCache) extends HellaCacheModule(outer) { assert(!s2_valid_hit) io.cpu.resp.valid := true io.cpu.resp.bits.replay := true + io.cpu.resp.bits.addr := s2_uncached_resp_addr } // load data subword mux/sign extension diff --git a/src/main/scala/rocket/Frontend.scala b/src/main/scala/rocket/Frontend.scala index 4784cd0e..f4d5ff78 100644 --- a/src/main/scala/rocket/Frontend.scala +++ b/src/main/scala/rocket/Frontend.scala @@ -42,34 +42,37 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) { val acquire = Bool(INPUT) } -class Frontend(implicit p: Parameters) extends LazyModule { +class Frontend(hartid: Int)(implicit p: Parameters) extends LazyModule { lazy val module = new FrontendModule(this) - val icache = LazyModule(new ICache(latency = 2)) - val node = TLOutputNode() + val icache = LazyModule(new ICache(latency = 2, hartid)) + val masterNode = TLOutputNode() + val slaveNode = TLInputNode() - node := icache.node + icache.slaveNode.map { _ := slaveNode } + masterNode := icache.masterNode } class FrontendBundle(outer: Frontend) extends CoreBundle()(outer.p) { val cpu = new FrontendIO().flip val ptw = new TLBPTWIO() - val mem = outer.node.bundleOut + val tl_out = outer.masterNode.bundleOut + val tl_in = outer.slaveNode.bundleIn val resetVector = UInt(INPUT, vaddrBitsExtended) + val hartid = UInt(INPUT, hartIdLen) } class FrontendModule(outer: Frontend) extends LazyModuleImp(outer) with HasCoreParameters with HasL1ICacheParameters { val io = new FrontendBundle(outer) - implicit val edge = outer.node.edgesOut(0) + implicit val edge = outer.masterNode.edgesOut.head val icache = outer.icache.module val tlb = Module(new TLB(log2Ceil(coreInstBytes*fetchWidth), nTLBEntries)) val fq = withReset(reset || io.cpu.req.valid) { Module(new ShiftQueue(new FrontendResp, 3, flow = true)) } val s0_valid = io.cpu.req.valid || fq.io.enq.ready - val s1_pc_ = Reg(UInt(width=vaddrBitsExtended)) - val s1_pc = ~(~s1_pc_ | (coreInstBytes-1)) // discard PC LSBS (this propagates down the pipeline) + val s1_pc = Reg(UInt(width=vaddrBitsExtended)) val s1_speculative = Reg(Bool()) val s2_valid = Reg(init=Bool(true)) val s2_pc = Reg(init=io.resetVector) @@ -94,7 +97,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer) s2_replay := (s2_valid && !fq.io.enq.fire()) || RegNext(s2_replay && !s0_valid) val npc = Mux(s2_replay, s2_pc, predicted_npc) - s1_pc_ := io.cpu.npc + s1_pc := io.cpu.npc // consider RVC fetches across blocks to be non-speculative if the first // part was non-speculative val s0_speculative = @@ -116,7 +119,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer) if (usingBTB) { val btb = Module(new BTB) btb.io.req.valid := false - btb.io.req.bits.addr := s1_pc_ + btb.io.req.bits.addr := s1_pc btb.io.btb_update := io.cpu.btb_update btb.io.bht_update := io.cpu.bht_update btb.io.ras_update := io.cpu.ras_update @@ -148,16 +151,18 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer) tlb.io.req.bits.sfence := io.cpu.sfence tlb.io.req.bits.size := log2Ceil(coreInstBytes*fetchWidth) + icache.io.hartid := io.hartid icache.io.req.valid := s0_valid icache.io.req.bits.addr := io.cpu.npc icache.io.invalidate := io.cpu.flush_icache icache.io.s1_paddr := tlb.io.resp.paddr + icache.io.s2_vaddr := s2_pc icache.io.s1_kill := io.cpu.req.valid || tlb.io.resp.miss || s2_replay icache.io.s2_kill := s2_speculative && !s2_cacheable || s2_xcpt fq.io.enq.valid := s2_valid && (icache.io.resp.valid || icache.io.s2_kill) fq.io.enq.bits.pc := s2_pc - io.cpu.npc := Mux(io.cpu.req.valid, io.cpu.req.bits.pc, npc) + io.cpu.npc := ~(~Mux(io.cpu.req.valid, io.cpu.req.bits.pc, npc) | (coreInstBytes-1)) // discard LSB(s) fq.io.enq.bits.data := icache.io.resp.bits fq.io.enq.bits.mask := UInt((1 << fetchWidth)-1) << s2_pc.extract(log2Ceil(fetchWidth)+log2Ceil(coreInstBytes)-1, log2Ceil(coreInstBytes)) @@ -170,14 +175,15 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer) io.cpu.resp <> fq.io.deq // performance events - io.cpu.acquire := edge.done(icache.io.mem(0).a) + io.cpu.acquire := edge.done(icache.io.tl_out(0).a) } /** Mix-ins for constructing tiles that have an ICache-based pipeline frontend */ trait HasICacheFrontend extends CanHavePTW with HasTileLinkMasterPort { val module: HasICacheFrontendModule - val frontend = LazyModule(new Frontend) - masterNode := frontend.node + val frontend = LazyModule(new Frontend(hartid: Int)) + val hartid: Int + masterNode := frontend.masterNode nPTWPorts += 1 } diff --git a/src/main/scala/rocket/HellaCache.scala b/src/main/scala/rocket/HellaCache.scala index 73e84d49..70072321 100644 --- a/src/main/scala/rocket/HellaCache.scala +++ b/src/main/scala/rocket/HellaCache.scala @@ -158,8 +158,8 @@ abstract class HellaCache(implicit p: Parameters) extends LazyModule { val module: HellaCacheModule } -class HellaCacheBundle(outer: HellaCache) extends Bundle { - implicit val p = outer.p +class HellaCacheBundle(outer: HellaCache)(implicit p: Parameters) extends CoreBundle()(p) { + val hartid = UInt(INPUT, hartIdLen) val cpu = (new HellaCacheIO).flip val ptw = new TLBPTWIO() val mem = outer.node.bundleOut @@ -184,7 +184,7 @@ object HellaCache { /** Mix-ins for constructing tiles that have a HellaCache */ -trait HasHellaCache extends HasTileLinkMasterPort { +trait HasHellaCache extends HasTileLinkMasterPort with HasTileParameters { val module: HasHellaCacheModule implicit val p: Parameters def findScratchpadFromICache: Option[AddressSet] diff --git a/src/main/scala/rocket/ICache.scala b/src/main/scala/rocket/ICache.scala index 160a472e..066b5411 100644 --- a/src/main/scala/rocket/ICache.scala +++ b/src/main/scala/rocket/ICache.scala @@ -19,6 +19,7 @@ case class ICacheParams( nTLBEntries: Int = 32, cacheIdBits: Int = 0, ecc: Option[Code] = None, + itimAddr: Option[BigInt] = None, blockBytes: Int = 64) extends L1CacheParams { def replacement = new RandomReplacement(nWays) } @@ -31,65 +32,114 @@ class ICacheReq(implicit p: Parameters) extends CoreBundle()(p) with HasL1ICache val addr = UInt(width = vaddrBits) } -class ICacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasL1ICacheParameters { - val data = Bits(width = coreInstBits) - val datablock = Bits(width = rowBits) -} - -class ICache(val latency: Int)(implicit p: Parameters) extends LazyModule { +class ICache(val latency: Int, val hartid: Int)(implicit p: Parameters) extends LazyModule + with HasRocketCoreParameters { lazy val module = new ICacheModule(this) - val node = TLClientNode(TLClientParameters(sourceId = IdRange(0,1))) + val masterNode = TLClientNode(TLClientParameters(sourceId = IdRange(0,1))) + + val icacheParams = tileParams.icache.get + val size = icacheParams.nSets * icacheParams.nWays * icacheParams.blockBytes + val slaveNode = icacheParams.itimAddr.map { itimAddr => + val wordBytes = coreInstBytes * fetchWidth + TLManagerNode(Seq(TLManagerPortParameters( + Seq(TLManagerParameters( + address = Seq(AddressSet(itimAddr, size-1)), + regionType = RegionType.UNCACHED, + executable = true, + supportsPutFull = TransferSizes(1, wordBytes), + supportsGet = TransferSizes(1, wordBytes), + fifoId = Some(0))), // requests handled in FIFO order + beatBytes = wordBytes, + minLatency = 1))) + } } class ICacheBundle(outer: ICache) extends CoreBundle()(outer.p) { - val req = Valid(new ICacheReq).flip + val hartid = UInt(INPUT, hartIdLen) + val req = Decoupled(new ICacheReq).flip val s1_paddr = UInt(INPUT, paddrBits) // delayed one cycle w.r.t. req + val s2_vaddr = UInt(INPUT, vaddrBits) // delayed two cycles w.r.t. req val s1_kill = Bool(INPUT) // delayed one cycle w.r.t. req val s2_kill = Bool(INPUT) // delayed two cycles; prevents I$ miss emission val resp = Valid(UInt(width = coreInstBits * fetchWidth)) val invalidate = Bool(INPUT) - val mem = outer.node.bundleOut + val tl_out = outer.masterNode.bundleOut + val tl_in = outer.slaveNode.map(_.bundleIn) +} + +// get a tile-specific property without breaking deduplication +object GetPropertyByHartId { + def apply[T <: Data](tiles: Seq[RocketTileParams], f: RocketTileParams => Option[T], hartId: UInt): T = { + PriorityMux(tiles.zipWithIndex.collect { case (t, i) if f(t).nonEmpty => (hartId === i) -> f(t).get }) + } } class ICacheModule(outer: ICache) extends LazyModuleImp(outer) with HasL1ICacheParameters { val io = new ICacheBundle(outer) - val edge = outer.node.edgesOut(0) - val tl_out = io.mem(0) + val edge_out = outer.masterNode.edgesOut.head + val tl_out = io.tl_out.head + val edge_in = outer.slaveNode.map(_.edgesIn.head) + val tl_in = io.tl_in.map(_.head) require(isPow2(nSets) && isPow2(nWays)) require(isPow2(coreInstBytes)) require(!usingVM || pgIdxBits >= untagBits) + val scratchpadOn = RegInit(false.B) + val scratchpadMax = tl_in.map(tl => Reg(UInt(width = log2Ceil(nSets * (nWays - 1))))) + def lineInScratchpad(line: UInt) = scratchpadMax.map(scratchpadOn && line <= _).getOrElse(false.B) + def addrMaybeInScratchpad(addr: UInt) = if (outer.icacheParams.itimAddr.isEmpty) false.B else { + val base = GetPropertyByHartId(p(coreplex.RocketTilesKey), _.icache.flatMap(_.itimAddr.map(_.U)), io.hartid) + addr >= base && addr < base + outer.size + } + def addrInScratchpad(addr: UInt) = addrMaybeInScratchpad(addr) && lineInScratchpad(addr(untagBits+log2Ceil(nWays)-1, blockOffBits)) + def scratchpadWay(addr: UInt) = addr.extract(untagBits+log2Ceil(nWays)-1, untagBits) + def scratchpadWayValid(way: UInt) = way < nWays - 1 + def scratchpadLine(addr: UInt) = addr(untagBits+log2Ceil(nWays)-1, blockOffBits) + val s0_slaveValid = tl_in.map(_.a.fire()).getOrElse(false.B) + val s1_slaveValid = RegNext(s0_slaveValid, false.B) + val s2_slaveValid = RegNext(s1_slaveValid, false.B) + val s3_slaveValid = RegNext(s2_slaveValid, false.B) + val s_ready :: s_request :: s_refill :: Nil = Enum(UInt(), 3) val state = Reg(init=s_ready) val invalidated = Reg(Bool()) val refill_addr = Reg(UInt(width = paddrBits)) + val refill_tag = refill_addr(tagBits+untagBits-1,untagBits) + val refill_idx = refill_addr(untagBits-1,blockOffBits) val s1_tag_hit = Wire(Vec(nWays, Bool())) - val s1_any_tag_hit = s1_tag_hit.reduce(_||_) + val s1_any_tag_hit = s1_tag_hit.reduce(_||_) || Mux(s1_slaveValid, true.B, addrMaybeInScratchpad(io.s1_paddr)) val s1_valid = Reg(init=Bool(false)) - val out_valid = s1_valid && state === s_ready && !io.s1_kill - val s1_idx = io.s1_paddr(untagBits-1,blockOffBits) - val s1_tag = io.s1_paddr(tagBits+untagBits-1,untagBits) - val s1_hit = out_valid && s1_any_tag_hit + val s1_hit = s1_valid && s1_any_tag_hit val s1_miss = s1_valid && state === s_ready && !s1_any_tag_hit - val s0_valid = io.req.valid && state === s_ready + io.req.ready := !(tl_out.d.fire() || s0_slaveValid || s3_slaveValid) + val s0_valid = io.req.fire() val s0_vaddr = io.req.bits.addr s1_valid := s0_valid when (s1_miss) { refill_addr := io.s1_paddr } - val refill_tag = refill_addr(tagBits+untagBits-1,untagBits) - val refill_idx = refill_addr(untagBits-1,blockOffBits) - val (_, _, refill_done, refill_cnt) = edge.count(tl_out.d) - tl_out.d.ready := Bool(true) - require (edge.manager.minLatency > 0) + val (_, _, refill_done, refill_cnt) = edge_out.count(tl_out.d) + tl_out.d.ready := !s3_slaveValid + require (edge_out.manager.minLatency > 0) + + val repl_way = if (isDM) UInt(0) else { + // pick a way that is not used by the scratchpad + val v0 = LFSR16(tl_out.a.fire())(log2Up(nWays)-1,0) + var v = v0 + for (i <- log2Ceil(nWays) - 1 to 0 by -1) { + val mask = nWays - (BigInt(1) << (i + 1)) + v = v | (lineInScratchpad(Cat(v0 | mask.U, refill_idx)) << i) + } + assert(!lineInScratchpad(Cat(v, refill_idx))) + v + } - val repl_way = if (isDM) UInt(0) else LFSR16(s1_miss)(log2Up(nWays)-1,0) val entagbits = code.width(tagBits) val tag_array = SeqMem(nSets, Vec(nWays, Bits(width = entagbits))) val tag_rdata = tag_array.read(s0_vaddr(untagBits-1,blockOffBits), !refill_done && s0_valid) @@ -99,8 +149,9 @@ class ICacheModule(outer: ICache) extends LazyModuleImp(outer) } val vb_array = Reg(init=Bits(0, nSets*nWays)) - when (refill_done && !invalidated) { - vb_array := vb_array.bitSet(Cat(repl_way, refill_idx), Bool(true)) + when (tl_out.d.fire()) { + // clear bit when refill starts so hit-under-miss doesn't fetch bad data + vb_array := vb_array.bitSet(Cat(repl_way, refill_idx), refill_done && !invalidated) } val invalidate = Wire(init = io.invalidate) when (invalidate) { @@ -111,72 +162,127 @@ class ICacheModule(outer: ICache) extends LazyModuleImp(outer) val s1_tag_disparity = Wire(Vec(nWays, Bool())) val wordBits = coreInstBits * fetchWidth val s1_dout = Wire(Vec(nWays, UInt(width = code.width(wordBits)))) - val s1_dout_valid = RegNext(s0_valid) + + val s0_slaveAddr = tl_in.map(_.a.bits.address).getOrElse(0.U) + val s1s3_slaveAddr = Reg(UInt(width = log2Ceil(outer.size))) + val s1s3_slaveData = Reg(UInt(width = wordBits)) for (i <- 0 until nWays) { - val s1_vb = vb_array(Cat(UInt(i), io.s1_paddr(untagBits-1,blockOffBits))).toBool - s1_tag_disparity(i) := code.decode(tag_rdata(i)).error - s1_tag_hit(i) := s1_vb && code.decode(tag_rdata(i)).uncorrected === s1_tag + val s1_idx = io.s1_paddr(untagBits-1,blockOffBits) + val s1_tag = io.s1_paddr(tagBits+untagBits-1,untagBits) + val scratchpadHit = scratchpadWayValid(i) && + Mux(s1_slaveValid, + lineInScratchpad(scratchpadLine(s1s3_slaveAddr)) && scratchpadWay(s1s3_slaveAddr) === i, + addrInScratchpad(io.s1_paddr) && scratchpadWay(io.s1_paddr) === i) + val s1_vb = vb_array(Cat(UInt(i), s1_idx)) && !s1_slaveValid + s1_tag_disparity(i) := s1_vb && code.decode(tag_rdata(i)).error + s1_tag_hit(i) := scratchpadHit || (s1_vb && code.decode(tag_rdata(i)).uncorrected === s1_tag) } + assert(!(s1_valid || s1_slaveValid) || PopCount(s1_tag_hit zip s1_tag_disparity map { case (h, d) => h && !d }) <= 1) - require(rowBits % wordBits == 0) - val data_arrays = Seq.fill(rowBits / wordBits) { SeqMem(nSets * refillCycles, Vec(nWays, UInt(width = code.width(wordBits)))) } + require(tl_out.d.bits.data.getWidth % wordBits == 0) + val data_arrays = Seq.fill(tl_out.d.bits.data.getWidth / wordBits) { SeqMem(nSets * refillCycles, Vec(nWays, UInt(width = code.width(wordBits)))) } for ((data_array, i) <- data_arrays zipWithIndex) { - val wen = tl_out.d.valid + def wordMatch(addr: UInt) = addr.extract(log2Ceil(tl_out.d.bits.data.getWidth/8)-1, log2Ceil(wordBits/8)) === i + def row(addr: UInt) = addr(untagBits-1, blockOffBits-log2Ceil(refillCycles)) + val s0_ren = (s0_valid && wordMatch(s0_vaddr)) || (s0_slaveValid && wordMatch(s0_slaveAddr)) + val wen = (tl_out.d.fire() && !invalidated) || (s3_slaveValid && wordMatch(s1s3_slaveAddr)) + val mem_idx = Mux(tl_out.d.fire(), (refill_idx << log2Ceil(refillCycles)) | refill_cnt, + Mux(s3_slaveValid, row(s1s3_slaveAddr), + Mux(s0_slaveValid, row(s0_slaveAddr), + row(s0_vaddr)))) when (wen) { - val idx = (refill_idx << log2Ceil(refillCycles)) | refill_cnt - val data = tl_out.d.bits.data(wordBits*(i+1)-1, wordBits*i) - data_array.write(idx, Vec.fill(nWays)(code.encode(data)), (0 until nWays).map(repl_way === _)) + val data = Mux(s3_slaveValid, s1s3_slaveData, tl_out.d.bits.data(wordBits*(i+1)-1, wordBits*i)) + val way = Mux(s3_slaveValid, scratchpadWay(s1s3_slaveAddr), repl_way) + data_array.write(mem_idx, Vec.fill(nWays)(code.encode(data)), (0 until nWays).map(way === _)) } - def wordMatch(addr: UInt) = addr.extract(log2Ceil(rowBytes)-1, log2Ceil(wordBits/8)) === i - val s0_raddr = s0_vaddr(untagBits-1,blockOffBits-log2Ceil(refillCycles)) - val dout = data_array.read(s0_raddr, !wen && (s0_valid && wordMatch(s0_vaddr))) - when (wordMatch(io.s1_paddr)) { + val dout = data_array.read(mem_idx, !wen && s0_ren) + when (wordMatch(Mux(s1_slaveValid, s1s3_slaveAddr, io.s1_paddr))) { s1_dout := dout } } -/* - for ((data_array, i) <- data_arrays zipWithIndex) { - val wen = tl_out.d.valid && repl_way === UInt(i) - when (wen) { - val e_d = code.encode(tl_out.d.bits.data) - data_array.write((refill_idx << log2Ceil(refillCycles)) | refill_cnt, e_d) - } - val s0_raddr = s0_vaddr(untagBits-1,blockOffBits-log2Ceil(refillCycles)) - s1_dout(i) := data_array.read(s0_raddr, !wen && s0_valid) - } -*/ - // output signals outer.latency match { case 1 => - require(code.width(rowBits) == rowBits) // no ECC + require(code.isInstanceOf[uncore.util.IdentityCode]) + require(outer.icacheParams.itimAddr.isEmpty) io.resp.bits := Mux1H(s1_tag_hit, s1_dout) io.resp.valid := s1_hit + case 2 => - val s2_valid = RegNext(out_valid, Bool(false)) + val s2_valid = RegNext(s1_valid && !io.s1_kill, Bool(false)) val s2_hit = RegNext(s1_hit, Bool(false)) - val s2_tag_hit = RegEnable(s1_tag_hit, s1_valid) - val s2_dout = RegEnable(s1_dout, s1_valid) + val s2_tag_hit = RegEnable(s1_tag_hit, s1_valid || s1_slaveValid) + val s2_dout = RegEnable(s1_dout, s1_valid || s1_slaveValid) val s2_way_mux = Mux1H(s2_tag_hit, s2_dout) - val s2_tag_disparity = RegEnable(s1_tag_disparity, s1_valid).asUInt.orR - val s2_data_disparity = code.decode(s2_way_mux).error - val s2_disparity = s2_tag_disparity || s2_data_disparity + val s2_tag_disparity = RegEnable(s1_tag_disparity, s1_valid || s1_slaveValid).asUInt.orR + val s2_data_decoded = code.decode(s2_way_mux) + val s2_disparity = s2_tag_disparity || s2_data_decoded.error when (s2_valid && s2_disparity) { invalidate := true } - io.resp.bits := code.decode(s2_way_mux).uncorrected + io.resp.bits := s2_data_decoded.uncorrected io.resp.valid := s2_hit && !s2_disparity + + tl_in.map { tl => + tl.a.ready := !tl_out.d.fire() && !s1_slaveValid && !s2_slaveValid && !(tl.d.valid && !tl.d.ready) + val s1_a = RegEnable(tl.a.bits, s0_slaveValid) + when (s0_slaveValid) { + val a = tl.a.bits + s1s3_slaveAddr := tl.a.bits.address + s1s3_slaveData := tl.a.bits.data + when (edge_in.get.hasData(a)) { + val enable = scratchpadWayValid(scratchpadWay(a.address)) + when (!lineInScratchpad(scratchpadLine(a.address))) { + scratchpadMax.get := scratchpadLine(a.address) + when (enable) { invalidate := true } + } + scratchpadOn := enable + } + } + + assert(!s2_valid || RegNext(RegNext(s0_vaddr)) === io.s2_vaddr) + when (!(tl.a.valid || s1_slaveValid || s2_slaveValid) + && s2_valid && s2_data_decoded.correctable && !s2_tag_disparity) { + // handle correctable errors on CPU accesses to the scratchpad. + // if there is an in-flight slave-port access to the scratchpad, + // report the a miss but don't correct the error (as there is + // a structural hazard on s1s3_slaveData/s1s3_slaveAddress). + s3_slaveValid := true + s1s3_slaveData := s2_data_decoded.corrected + s1s3_slaveAddr := Cat(OHToUInt(s2_tag_hit), io.s2_vaddr(untagBits-1, log2Ceil(wordBits/8)), s1s3_slaveAddr(log2Ceil(wordBits/8)-1, 0)) + } + + val respValid = RegInit(false.B) + respValid := s2_slaveValid || (respValid && !tl.d.ready) + when (s2_slaveValid) { + when (edge_in.get.hasData(s1_a)) { s3_slaveValid := true } + def byteEn(i: Int) = !(edge_in.get.hasData(s1_a) && s1_a.mask(i)) + s1s3_slaveData := (0 until wordBits/8).map(i => Mux(byteEn(i), s2_data_decoded.corrected, s1s3_slaveData)(8*(i+1)-1, 8*i)).asUInt + } + + tl.d.valid := respValid + tl.d.bits := Mux(edge_in.get.hasData(s1_a), + edge_in.get.AccessAck(s1_a, UInt(0)), + edge_in.get.AccessAck(s1_a, UInt(0), UInt(0))) + tl.d.bits.data := s1s3_slaveData + + // Tie off unused channels + tl.b.valid := false + tl.c.ready := true + tl.e.ready := true + } } tl_out.a.valid := state === s_request && !io.s2_kill - tl_out.a.bits := edge.Get( + tl_out.a.bits := edge_out.Get( fromSource = UInt(0), toAddress = (refill_addr >> blockOffBits) << blockOffBits, lgSize = lgCacheBlockBytes)._2 tl_out.b.ready := Bool(true) tl_out.c.valid := Bool(false) tl_out.e.valid := Bool(false) + assert(!(tl_out.a.valid && addrMaybeInScratchpad(tl_out.a.bits.address))) // control state machine switch (state) { diff --git a/src/main/scala/rocket/PMP.scala b/src/main/scala/rocket/PMP.scala index cec4f25e..3337da0a 100644 --- a/src/main/scala/rocket/PMP.scala +++ b/src/main/scala/rocket/PMP.scala @@ -42,7 +42,7 @@ class PMP(implicit p: Parameters) extends PMPReg { val mask = UInt(width = paddrBits) import PMP._ - def computeMask = Cat((Cat(addr, cfg.a(0)) + 1) ^ Cat(addr, cfg.a(0)), UInt((BigInt(1) << (lgAlign-1)) - 1, lgAlign-1)) + def computeMask = Cat(Cat(addr, cfg.a(0)) & ~(Cat(addr, cfg.a(0)) + 1), UInt((BigInt(1) << lgAlign) - 1, lgAlign)) private def comparand = addr << lgAlign private def pow2Match(x: UInt, lgSize: UInt, lgMaxSize: Int) = { diff --git a/src/main/scala/rocket/Rocket.scala b/src/main/scala/rocket/Rocket.scala index 853fe734..112b4d6c 100644 --- a/src/main/scala/rocket/Rocket.scala +++ b/src/main/scala/rocket/Rocket.scala @@ -8,8 +8,10 @@ import chisel3.core.withReset import config._ import tile._ import uncore.constants._ +import diplomacy._ import util._ import Chisel.ImplicitConversions._ +import collection.immutable.ListMap case class RocketCoreParams( bootFreqHz: BigInt = 0, diff --git a/src/main/scala/rocket/ScratchpadSlavePort.scala b/src/main/scala/rocket/ScratchpadSlavePort.scala index 8aee7658..a96fd248 100644 --- a/src/main/scala/rocket/ScratchpadSlavePort.scala +++ b/src/main/scala/rocket/ScratchpadSlavePort.scala @@ -105,17 +105,25 @@ class ScratchpadSlavePort(address: AddressSet)(implicit p: Parameters) extends L } /** Mix-ins for constructing tiles that have optional scratchpads */ -trait CanHaveScratchpad extends HasHellaCache with HasICacheFrontend { +trait CanHaveScratchpad extends HasHellaCache with HasICacheFrontend with HasCoreParameters { val module: CanHaveScratchpadModule + val slaveNode = TLInputNode() // Up to two uses for this input node: + + // 1) Frontend always exists, but may or may not have a scratchpad node + val fg = LazyModule(new TLFragmenter(fetchWidth*coreInstBytes, p(CacheBlockBytes), true)) + val ww = LazyModule(new TLWidthWidget(xLen/8)) + frontend.slaveNode :*= fg.node + fg.node :*= ww.node + ww.node :*= slaveNode + + // 2) ScratchpadSlavePort always has a node, but only exists when the HellaCache has a scratchpad val scratch = tileParams.dcache.flatMap(d => d.scratch.map(s => LazyModule(new ScratchpadSlavePort(AddressSet(s, d.dataScratchpadBytes-1))))) - val slaveNode = TLInputNode() - - scratch foreach { lm => lm.node := TLFragmenter(p(XLen)/8, p(CacheBlockBytes))(slaveNode) } + scratch foreach { lm => lm.node := TLFragmenter(xLen/8, p(CacheBlockBytes))(slaveNode) } def findScratchpadFromICache: Option[AddressSet] = scratch.map { s => - val finalNode = frontend.node.edgesOut(0).manager.managers.find(_.nodePath.last == s.node) + val finalNode = frontend.masterNode.edgesOut.head.manager.managers.find(_.nodePath.last == s.node) require (finalNode.isDefined, "Could not find the scratch pad; not reachable via icache?") require (finalNode.get.address.size == 1, "Scratchpad address space was fragmented!") finalNode.get.address(0) diff --git a/src/main/scala/rocket/Tile.scala b/src/main/scala/rocket/Tile.scala index 97f8357f..8691d9a3 100644 --- a/src/main/scala/rocket/Tile.scala +++ b/src/main/scala/rocket/Tile.scala @@ -132,6 +132,8 @@ class RocketTileModule(outer: RocketTile) extends BaseTileModule(outer, () => ne core.io.hartid := io.hartid // Pass through the hartid outer.frontend.module.io.cpu <> core.io.imem outer.frontend.module.io.resetVector := io.resetVector + outer.frontend.module.io.hartid := io.hartid + outer.dcache.module.io.hartid := io.hartid dcachePorts += core.io.dmem // TODO outer.dcachePorts += () => module.core.io.dmem ?? fpuOpt foreach { fpu => core.io.fpu <> fpu.io } core.io.ptw <> ptw.io.dpath @@ -174,12 +176,10 @@ class SyncRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Parameters) xing.intnode := intNode lazy val module = new LazyModuleImp(this) { - val io = new Bundle { + val io = new CoreBundle with HasExternallyDrivenTileConstants { val master = masterNode.bundleOut val slave = slaveNode.bundleIn val interrupts = intNode.bundleIn - val hartid = UInt(INPUT, p(XLen)) - val resetVector = UInt(INPUT, p(XLen)) } // signals that do not change: rocket.module.io.hartid := io.hartid @@ -206,12 +206,10 @@ class AsyncRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Parameters xing.intnode := intNode lazy val module = new LazyModuleImp(this) { - val io = new Bundle { + val io = new CoreBundle with HasExternallyDrivenTileConstants { val master = masterNode.bundleOut val slave = slaveNode.bundleIn val interrupts = intNode.bundleIn - val hartid = UInt(INPUT, p(XLen)) - val resetVector = UInt(INPUT, p(XLen)) } // signals that do not change: rocket.module.io.hartid := io.hartid @@ -242,12 +240,10 @@ class RationalRocketTile(rtp: RocketTileParams, hartid: Int)(implicit p: Paramet xing.intnode := intNode lazy val module = new LazyModuleImp(this) { - val io = new Bundle { + val io = new CoreBundle with HasExternallyDrivenTileConstants { val master = masterNode.bundleOut val slave = slaveNode.bundleIn val interrupts = intNode.bundleIn - val hartid = UInt(INPUT, p(XLen)) - val resetVector = UInt(INPUT, p(XLen)) } // signals that do not change: rocket.module.io.hartid := io.hartid diff --git a/src/main/scala/rocketchip/RocketPlexMaster.scala b/src/main/scala/rocketchip/RocketPlexMaster.scala index d0a31dd1..54731d56 100644 --- a/src/main/scala/rocketchip/RocketPlexMaster.scala +++ b/src/main/scala/rocketchip/RocketPlexMaster.scala @@ -3,17 +3,13 @@ package rocketchip import Chisel._ -import config._ -import diplomacy._ -import uncore.tilelink2._ -import uncore.devices._ -import util._ -import coreplex._ +import coreplex.RocketPlex +import diplomacy.LazyModule trait RocketPlexMaster extends HasTopLevelNetworks { val module: RocketPlexMasterModule - val coreplex = LazyModule(new DefaultCoreplex) + val coreplex = LazyModule(new RocketPlex) coreplex.l2in :=* fsb.node bsb.node :*= coreplex.l2out diff --git a/src/main/scala/tile/BaseTile.scala b/src/main/scala/tile/BaseTile.scala index bebe2bfb..3aef5be2 100644 --- a/src/main/scala/tile/BaseTile.scala +++ b/src/main/scala/tile/BaseTile.scala @@ -11,6 +11,8 @@ import util._ case object SharedMemoryTLEdge extends Field[TLEdgeOut] case object TileKey extends Field[TileParams] +case object ResetVectorBits extends Field[Int] +case object MaxHartIdBits extends Field[Int] trait TileParams { val core: CoreParams @@ -31,6 +33,7 @@ trait HasTileParameters { val usingBTB = tileParams.btb.isDefined && tileParams.btb.get.nEntries > 0 val usingPTW = usingVM val usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined + val hartIdLen = p(MaxHartIdBits) def dcacheArbPorts = 1 + usingVM.toInt + usingDataScratchpad.toInt + tileParams.rocc.size } @@ -47,8 +50,8 @@ abstract class BareTileModule[+L <: BareTile, +B <: BareTileBundle[L]](_outer: L val io = _io () } -// Uses TileLink master port to connect caches and accelerators to the coreplex -trait HasTileLinkMasterPort extends HasTileParameters { +/** Uses TileLink master port to connect caches and accelerators to the coreplex */ +trait HasTileLinkMasterPort { implicit val p: Parameters val module: HasTileLinkMasterPortModule val masterNode = TLOutputNode() @@ -64,6 +67,14 @@ trait HasTileLinkMasterPortModule { val io: HasTileLinkMasterPortBundle } +/** Some other standard inputs */ +trait HasExternallyDrivenTileConstants extends Bundle { + implicit val p: Parameters + val hartid = UInt(INPUT, p(MaxHartIdBits)) + val resetVector = UInt(INPUT, p(ResetVectorBits)) +} + +/** Base class for all Tiles that use TileLink */ abstract class BaseTile(tileParams: TileParams)(implicit p: Parameters) extends BareTile with HasTileLinkMasterPort with HasExternalInterrupts { @@ -71,11 +82,10 @@ abstract class BaseTile(tileParams: TileParams)(implicit p: Parameters) extends } class BaseTileBundle[+L <: BaseTile](_outer: L) extends BareTileBundle(_outer) + with HasTileParameters with HasTileLinkMasterPortBundle - with HasExternalInterruptsBundle { - val hartid = UInt(INPUT, p(XLen)) - val resetVector = UInt(INPUT, p(XLen)) -} + with HasExternalInterruptsBundle + with HasExternallyDrivenTileConstants class BaseTileModule[+L <: BaseTile, +B <: BaseTileBundle[L]](_outer: L, _io: () => B) extends BareTileModule(_outer, _io) with HasTileLinkMasterPortModule diff --git a/src/main/scala/tile/Core.scala b/src/main/scala/tile/Core.scala index a17544e6..2befafad 100644 --- a/src/main/scala/tile/Core.scala +++ b/src/main/scala/tile/Core.scala @@ -75,13 +75,12 @@ abstract class CoreModule(implicit val p: Parameters) extends Module abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCoreParameters -trait HasCoreIO { +trait HasCoreIO extends HasTileParameters { implicit val p: Parameters - val io = new Bundle { + val io = new CoreBundle()(p) with HasExternallyDrivenTileConstants { val interrupts = new TileInterrupts().asInput - val hartid = UInt(INPUT, p(XLen)) - val imem = new FrontendIO()(p) - val dmem = new HellaCacheIO()(p) + val imem = new FrontendIO + val dmem = new HellaCacheIO val ptw = new DatapathPTWIO().flip val fpu = new FPUCoreIO().flip val rocc = new RoCCCoreIO().flip diff --git a/src/main/scala/uncore/devices/Plic.scala b/src/main/scala/uncore/devices/Plic.scala index 5507eb1a..0c0e26aa 100644 --- a/src/main/scala/uncore/devices/Plic.scala +++ b/src/main/scala/uncore/devices/Plic.scala @@ -12,6 +12,7 @@ import uncore.tilelink2._ import config._ import scala.math.min import tile.XLen +import util._ class GatewayPLICIO extends Bundle { val valid = Bool(OUTPUT) @@ -76,7 +77,7 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame device = device, beatBytes = p(XLen)/8, undefZero = false, - concurrency = 1) // Work around the enable -> claim hazard + concurrency = 1) // limiting concurrency handles RAW hazards on claim registers val intnode = IntNexusNode( numSourcePorts = 0 to 1024, @@ -128,11 +129,11 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame require(nHarts > 0 && nHarts <= PLICConsts.maxHarts) // For now, use LevelGateways for all TL2 interrupts - val gateways = Vec(interrupts.map { case i => + val gateways = Vec((false.B +: interrupts).map { case i => val gateway = Module(new LevelGateway) gateway.io.interrupt := i gateway.io.plic - } ++ (if (interrupts.isEmpty) Some(Wire(new GatewayPLICIO)) else None)) + }) val priority = if (nPriorities > 0) Reg(Vec(nDevices+1, UInt(width=log2Up(nPriorities+1)))) @@ -143,7 +144,7 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame val pending = Reg(init=Vec.fill(nDevices+1){Bool(false)}) val enables = Reg(Vec(nHarts, Vec(nDevices+1, Bool()))) - for ((p, g) <- pending.tail zip gateways) { + for ((p, g) <- pending zip gateways) { g.ready := !p g.complete := false when (g.valid) { p := true } @@ -152,19 +153,18 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame def findMax(x: Seq[UInt]): (UInt, UInt) = { if (x.length > 1) { val half = 1 << (log2Ceil(x.length) - 1) - val lMax = findMax(x take half) - val rMax = findMax(x drop half) - val useLeft = lMax._1 >= rMax._1 - (Mux(useLeft, lMax._1, rMax._1), Mux(useLeft, lMax._2, UInt(half) | rMax._2)) + val left = findMax(x take half) + val right = findMax(x drop half) + MuxT(left._1 >= right._1, left, (right._1, UInt(half) | right._2)) } else (x.head, UInt(0)) } val maxDevs = Reg(Vec(nHarts, UInt(width = log2Up(pending.size)))) for (hart <- 0 until nHarts) { - val effectivePriority = - for (((p, en), pri) <- (pending zip enables(hart) zip priority).tail) - yield Cat(p && en, pri) - val (maxPri, maxDev) = findMax((UInt(1) << priority(0).getWidth) +: effectivePriority) + val effectivePriority = (UInt(1) << priority(0).getWidth) +: + (for (((p, en), pri) <- (pending zip enables(hart) zip priority).tail) + yield Cat(p && en, pri)) + val (maxPri, maxDev) = findMax(effectivePriority) maxDevs(hart) := maxDev harts(hart) := Reg(next = maxPri) > Cat(UInt(1), threshold(hart)) @@ -185,13 +185,13 @@ class TLPLIC(maxPriorities: Int, address: BigInt = 0xC000000)(implicit p: Parame RegReadFn { valid => when (valid) { pending(maxDevs(i)) := Bool(false) - maxDevs(i) := UInt(0) // flush pipeline } (Bool(true), maxDevs(i)) }, RegWriteFn { (valid, data) => - when (valid && enables(i)(data)) { - gateways(data - UInt(1)).complete := Bool(true) + val irq = data.extract(log2Ceil(nDevices+1)-1, 0) + when (valid && enables(i)(irq)) { + gateways(irq).complete := Bool(true) } Bool(true) } diff --git a/src/main/scala/uncore/util/Ecc.scala b/src/main/scala/uncore/util/Ecc.scala index bfaaed12..b468d50d 100644 --- a/src/main/scala/uncore/util/Ecc.scala +++ b/src/main/scala/uncore/util/Ecc.scala @@ -38,10 +38,10 @@ class ParityCode extends Code def width(w0: Int) = w0+1 def encode(x: UInt) = Cat(x.xorR, x) def decode(y: UInt) = new Decoding { - def uncorrected = y(y.getWidth-2,0) - def corrected = uncorrected - def correctable = Bool(false) - def uncorrectable = y.xorR + val uncorrected = y(y.getWidth-2,0) + val corrected = uncorrected + val correctable = Bool(false) + val uncorrectable = y.xorR } } @@ -78,10 +78,10 @@ class SECCode extends Code }).asUInt private def swizzle(z: UInt) = (1 to n).filter(i => !isPow2(i)).map(i => z(i-1)).asUInt - def uncorrected = swizzle(y) - def corrected = swizzle(((y << 1) ^ UIntToOH(syndrome)) >> 1) - def correctable = syndrome.orR - def uncorrectable = syndrome > UInt(n) + val uncorrected = swizzle(y) + val corrected = swizzle(((y << 1) ^ UIntToOH(syndrome)) >> 1) + val correctable = syndrome.orR + val uncorrectable = syndrome > UInt(n) } private def mapping(i: Int) = i-1-log2Up(i) } @@ -97,10 +97,10 @@ class SECDEDCode extends Code val secdec = sec.decode(x(x.getWidth-2,0)) val pardec = par.decode(x) - def uncorrected = secdec.uncorrected - def corrected = secdec.corrected - def correctable = pardec.uncorrectable - def uncorrectable = !pardec.uncorrectable && secdec.correctable + val uncorrected = secdec.uncorrected + val corrected = secdec.corrected + val correctable = pardec.uncorrectable + val uncorrectable = !pardec.uncorrectable && secdec.correctable } } diff --git a/src/main/scala/util/Package.scala b/src/main/scala/util/Package.scala index 85667677..33907996 100644 --- a/src/main/scala/util/Package.scala +++ b/src/main/scala/util/Package.scala @@ -11,7 +11,9 @@ package object util { implicit class SeqToAugmentedSeq[T <: Data](val x: Seq[T]) extends AnyVal { def apply(idx: UInt): T = { - if (!isPow2(x.size)) { + if (x.size == 1) { + x.head + } else if (!isPow2(x.size)) { // For non-power-of-2 seqs, reflect elements to simplify decoder (x ++ x.takeRight(x.size & -x.size)).toSeq(idx) } else {