Merge branch 'ss-frontend'

rocket/src/main/scala/btb.scala

@@ -45,6 +45,15 @@ class BHTResp extends Bundle with BTBParameters {
   val value = UInt(width = 2)
 }
 
+// BHT contains table of 2-bit counters and a global history register.
+// The BHT only predicts and updates when there is a BTB hit.
+// The global history:
+//    - updated speculatively in fetch (if there's a BTB hit).
+//    - on a mispredict, the history register is reset (again, only if BTB hit).
+// The counter table:
+//    - each counter corresponds with the address of the fetch packet ("fetch pc").
+//    - updated when a branch resolves (and BTB was a hit for that branch).
+//      The updating branch must provide its "fetch pc".
 class BHT(nbht: Int) {
   val nbhtbits = log2Up(nbht)
   def get(addr: UInt, update: Bool): BHTResp = {
@@ -66,20 +75,43 @@ class BHT(nbht: Int) {
   val history = Reg(UInt(width = nbhtbits))
 }
 
+// BTB update occurs during branch resolution (and only on a mispredict).
+//  - "pc" is what future fetch PCs will tag match against.
+//  - "br_pc" is the PC of the branch instruction.
 class BTBUpdate extends Bundle with BTBParameters {
   val prediction = Valid(new BTBResp)
   val pc = UInt(width = vaddrBits)
   val target = UInt(width = vaddrBits)
-  val returnAddr = UInt(width = vaddrBits)
   val taken = Bool()
   val isJump = Bool()
-  val isCall = Bool()
   val isReturn = Bool()
+  val br_pc = UInt(width = vaddrBits)
+}
+
+// BHT update occurs during branch resolution on all conditional branches.
+//  - "pc" is what future fetch PCs will tag match against.
+class BHTUpdate extends Bundle with BTBParameters {
+  val prediction = Valid(new BTBResp)
+  val pc = UInt(width = vaddrBits)
+  val taken = Bool()
   val mispredict = Bool()
 }
 
+class RASUpdate extends Bundle with BTBParameters {
+  val isCall = Bool()
+  val isReturn = Bool()
+  val returnAddr = UInt(width = vaddrBits)
+  val prediction = Valid(new BTBResp)
+}
+
+//  - "bridx" is the low-order PC bits of the predicted branch (after
+//     shifting off the lowest log(inst_bytes) bits off).
+//  - "resp.mask" provides a mask of valid instructions (instructions are
+//     masked off by the predicted taken branch).
 class BTBResp extends Bundle with BTBParameters {
   val taken = Bool()
+  val mask = Bits(width = params(FetchWidth))
+  val bridx = Bits(width = log2Up(params(FetchWidth)))
   val target = UInt(width = vaddrBits)
   val entry = UInt(width = opaqueBits)
   val bht = new BHTResp
@@ -90,11 +122,16 @@ class BTBReq extends Bundle with BTBParameters {
 }
 
 // fully-associative branch target buffer
-class BTB extends Module with BTBParameters {
+// Higher-performance processors may cause BTB updates to occur out-of-order,
+// which requires an extra CAM port for updates (to ensure no duplicates get
+// placed in BTB).
+class BTB(updates_out_of_order: Boolean = false) extends Module with BTBParameters {
   val io = new Bundle {
     val req = Valid(new BTBReq).flip
     val resp = Valid(new BTBResp)
-    val update = Valid(new BTBUpdate).flip
+    val btb_update = Valid(new BTBUpdate).flip
+    val bht_update = Valid(new BHTUpdate).flip
+    val ras_update = Valid(new RASUpdate).flip
     val invalidate = Bool(INPUT)
   }
 
@@ -110,6 +147,7 @@ class BTB extends Module with BTBParameters {
 
   val useRAS = Reg(UInt(width = entries))
   val isJump = Reg(UInt(width = entries))
+  val brIdx  = Mem(UInt(width=log2Up(params(FetchWidth))), entries)
 
   private def page(addr: UInt) = addr >> matchBits
   private def pageMatch(addr: UInt) = {
@@ -123,58 +161,62 @@ class BTB extends Module with BTBParameters {
     idxValid & idxMatch & idxPageMatch
   }
 
-  val r_update = Pipe(io.update)
+  val r_btb_update = Pipe(io.btb_update)
   val update_target = io.req.bits.addr
 
   val pageHit = pageMatch(io.req.bits.addr)
   val hits = tagMatch(io.req.bits.addr, pageHit)
-  val updatePageHit = pageMatch(r_update.bits.pc)
+  val updatePageHit = pageMatch(r_btb_update.bits.pc)
-  val updateHits = tagMatch(r_update.bits.pc, updatePageHit)
+  val updateHits = tagMatch(r_btb_update.bits.pc, updatePageHit)
 
-  private var lfsr = LFSR16(r_update.valid)
+  private var lfsr = LFSR16(r_btb_update.valid)
   def rand(width: Int) = {
     lfsr = lfsr(lfsr.getWidth-1,1)
     Random.oneHot(width, lfsr)
   }
 
-  val updateHit = r_update.bits.prediction.valid
+  val updateHit = r_btb_update.bits.prediction.valid
-  val updateValid = r_update.bits.mispredict || updateHit && Bool(nBHT > 0)
-  val updateTarget = updateValid && r_update.bits.mispredict && r_update.bits.taken
 
   val useUpdatePageHit = updatePageHit.orR
-  val doIdxPageRepl = updateTarget && !useUpdatePageHit
+  val doIdxPageRepl = !useUpdatePageHit
   val idxPageRepl = UInt()
   val idxPageUpdateOH = Mux(useUpdatePageHit, updatePageHit, idxPageRepl)
   val idxPageUpdate = OHToUInt(idxPageUpdateOH)
   val idxPageReplEn = Mux(doIdxPageRepl, idxPageRepl, UInt(0))
 
-  val samePage = page(r_update.bits.pc) === page(update_target)
+  val samePage = page(r_btb_update.bits.pc) === page(update_target)
   val usePageHit = (pageHit & ~idxPageReplEn).orR
-  val doTgtPageRepl = updateTarget && !samePage && !usePageHit
+  val doTgtPageRepl = !samePage && !usePageHit
   val tgtPageRepl = Mux(samePage, idxPageUpdateOH, idxPageUpdateOH(nPages-2,0) << 1 | idxPageUpdateOH(nPages-1))
   val tgtPageUpdate = OHToUInt(Mux(usePageHit, pageHit, tgtPageRepl))
   val tgtPageReplEn = Mux(doTgtPageRepl, tgtPageRepl, UInt(0))
   val doPageRepl = doIdxPageRepl || doTgtPageRepl
 
   val pageReplEn = idxPageReplEn | tgtPageReplEn
-  idxPageRepl := UIntToOH(Counter(r_update.valid && doPageRepl, nPages)._1)
+  idxPageRepl := UIntToOH(Counter(r_btb_update.valid && doPageRepl, nPages)._1)
 
-  when (r_update.valid && !(updateValid && !updateTarget)) {
+  when (r_btb_update.valid) {
-    val nextRepl = Counter(!updateHit && updateValid, entries)._1
+    assert(io.req.bits.addr === r_btb_update.bits.target, "BTB request != I$ target")
-    val waddr = Mux(updateHit, r_update.bits.prediction.bits.entry, nextRepl)
+
+    val nextRepl = Counter(!updateHit, entries)._1
+    val waddr =
+      if (updates_out_of_order) Mux(updateHits.orR, OHToUInt(updateHits), nextRepl)
+      else Mux(updateHit, r_btb_update.bits.prediction.bits.entry, nextRepl)
 
     // invalidate entries if we stomp on pages they depend upon
     idxValid := idxValid & ~Vec.tabulate(entries)(i => (pageReplEn & (idxPagesOH(i) | tgtPagesOH(i))).orR).toBits
 
-    idxValid(waddr) := updateValid
+    idxValid(waddr) := Bool(true)
-    when (updateTarget) {
+    idxs(waddr) := r_btb_update.bits.pc
-      assert(io.req.bits.addr === r_update.bits.target, "BTB request != I$ target")
+    tgts(waddr) := update_target
-      idxs(waddr) := r_update.bits.pc
+    idxPages(waddr) := idxPageUpdate
-      tgts(waddr) := update_target
+    tgtPages(waddr) := tgtPageUpdate
-      idxPages(waddr) := idxPageUpdate
+    useRAS(waddr) := r_btb_update.bits.isReturn
-      tgtPages(waddr) := tgtPageUpdate
+    isJump(waddr) := r_btb_update.bits.isJump
-      useRAS(waddr) := r_update.bits.isReturn
+    if (params(FetchWidth) == 1) {
-      isJump(waddr) := r_update.bits.isJump
+      brIdx(waddr) := UInt(0)
+    } else {
+      brIdx(waddr) := r_btb_update.bits.br_pc >> log2Up(params(CoreInstBits)/8)
     }
 
     require(nPages % 2 == 0)
@@ -185,9 +227,9 @@ class BTB extends Module with BTBParameters {
         when (en && pageReplEn(i)) { pages(i) := data }
 
     writeBank(0, 2, Mux(idxWritesEven, doIdxPageRepl, doTgtPageRepl),
-      Mux(idxWritesEven, page(r_update.bits.pc), page(update_target)))
+      Mux(idxWritesEven, page(r_btb_update.bits.pc), page(update_target)))
     writeBank(1, 2, Mux(idxWritesEven, doTgtPageRepl, doIdxPageRepl),
-      Mux(idxWritesEven, page(update_target), page(r_update.bits.pc)))
+      Mux(idxWritesEven, page(update_target), page(r_btb_update.bits.pc)))
 
     when (doPageRepl) { pageValid := pageValid | pageReplEn }
   }
@@ -201,14 +243,21 @@ class BTB extends Module with BTBParameters {
   io.resp.bits.taken := io.resp.valid
   io.resp.bits.target := Cat(Mux1H(Mux1H(hits, tgtPagesOH), pages), Mux1H(hits, tgts))
   io.resp.bits.entry := OHToUInt(hits)
+  io.resp.bits.bridx := brIdx(io.resp.bits.entry)
+  if (params(FetchWidth) == 1) {
+    io.resp.bits.mask := UInt(1)
+  } else {
+    // note: btb_resp is clock gated, so the mask is only relevant for the io.resp.valid case
+    io.resp.bits.mask := Mux(io.resp.bits.taken, Cat((UInt(1) << brIdx(io.resp.bits.entry))-1, UInt(1)),
+                                                 SInt(-1))
+  }
 
   if (nBHT > 0) {
     val bht = new BHT(nBHT)
     val res = bht.get(io.req.bits.addr, io.req.valid && hits.orR && !Mux1H(hits, isJump))
-    val update_btb_hit = io.update.bits.prediction.valid
+    val update_btb_hit = io.bht_update.bits.prediction.valid
-    when (io.update.valid && update_btb_hit && !io.update.bits.isJump) {
+    when (io.bht_update.valid && update_btb_hit) {
-      bht.update(io.update.bits.pc, io.update.bits.prediction.bits.bht,
+      bht.update(io.bht_update.bits.pc, io.bht_update.bits.prediction.bits.bht, io.bht_update.bits.taken, io.bht_update.bits.mispredict)
-                 io.update.bits.taken, io.update.bits.mispredict)
     }
     when (!res.value(0) && !Mux1H(hits, isJump)) { io.resp.bits.taken := false }
     io.resp.bits.bht := res
@@ -220,13 +269,13 @@ class BTB extends Module with BTBParameters {
     when (!ras.isEmpty && doPeek) {
       io.resp.bits.target := ras.peek
     }
-    when (io.update.valid) {
+    when (io.ras_update.valid) {
-      when (io.update.bits.isCall) {
+      when (io.ras_update.bits.isCall) {
-        ras.push(io.update.bits.returnAddr)
+        ras.push(io.ras_update.bits.returnAddr)
         when (doPeek) {
-          io.resp.bits.target := io.update.bits.returnAddr
+          io.resp.bits.target := io.ras_update.bits.returnAddr
         }
-      }.elsewhen (io.update.bits.isReturn && io.update.bits.prediction.valid) {
+      }.elsewhen (io.ras_update.bits.isReturn && io.ras_update.bits.prediction.valid) {
         ras.pop
       }
     }

rocket/src/main/scala/core.scala

@@ -9,6 +9,7 @@ import uncore._
 case object BuildFPU extends Field[Option[() => FPU]]
 case object XprLen extends Field[Int]
 case object NMultXpr extends Field[Int]
+case object FetchWidth extends Field[Int]
 case object RetireWidth extends Field[Int]
 case object UseVM extends Field[Boolean]
 case object FastLoadWord extends Field[Boolean]
@@ -20,6 +21,7 @@ case object CoreDCacheReqTagBits extends Field[Int]
 
 abstract trait CoreParameters extends UsesParameters {
   val xprLen = params(XprLen)
+  val coreFetchWidth = params(FetchWidth)
   val coreInstBits = params(CoreInstBits)
   val coreInstBytes = coreInstBits/8
   val coreDataBits = xprLen
@@ -32,6 +34,7 @@ abstract trait CoreParameters extends UsesParameters {
 
 abstract trait RocketCoreParameters extends CoreParameters
 {
+  require(params(FetchWidth) == 1)  // for now...
   require(params(RetireWidth) == 1) // for now...
 }
 

rocket/src/main/scala/ctrl.scala

@@ -60,7 +60,7 @@ abstract trait DecodeConstants
                 //   |     | | | | | | | |       |       |      |      |         | |         |     | | | | | | csr   | | | | amo
                 //   |     | | | | | | | |       |       |      |      |         | |         |     | | | | | | |     | | | | |
                 List(N,    X,X,X,X,X,X,X,A2_X,   A1_X,   IMM_X, DW_X,  FN_X,     N,M_X,      MT_X, X,X,X,X,X,X,CSR.X,X,X,X,X,X)
-                                        
+
   val table: Array[(UInt, List[UInt])]
 }
 
@@ -505,7 +505,7 @@ class Control extends Module
   when (mem_xcpt) { wb_reg_cause := mem_cause }
 
   val wb_set_sboard = wb_ctrl.div || wb_dcache_miss || wb_ctrl.rocc
-  val replay_wb_common = 
+  val replay_wb_common =
     io.dmem.resp.bits.nack || wb_reg_replay || io.dpath.csr_replay
   val wb_rocc_val = wb_reg_valid && wb_ctrl.rocc && !replay_wb_common
   val replay_wb = replay_wb_common || wb_reg_valid && wb_ctrl.rocc && !io.rocc.cmd.ready
@@ -558,14 +558,24 @@ class Control extends Module
     Mux(wb_reg_valid && wb_ctrl.sret, PC_PCR, // sret instruction
                                       PC_MEM)))
 
-  io.imem.btb_update.valid := mem_reg_valid && (mem_ctrl.branch || io.imem.btb_update.bits.isJump) && !take_pc_wb
+  io.imem.btb_update.valid := mem_reg_valid && io.dpath.mem_misprediction && ((mem_ctrl.branch && io.dpath.mem_br_taken) || mem_ctrl.jalr || mem_ctrl.jal) && !take_pc_wb
   io.imem.btb_update.bits.prediction.valid := mem_reg_btb_hit
   io.imem.btb_update.bits.prediction.bits := mem_reg_btb_resp
-  io.imem.btb_update.bits.taken := mem_ctrl.branch && io.dpath.mem_br_taken || io.imem.btb_update.bits.isJump
-  io.imem.btb_update.bits.mispredict := mem_misprediction
   io.imem.btb_update.bits.isJump := mem_ctrl.jal || mem_ctrl.jalr
-  io.imem.btb_update.bits.isCall := mem_ctrl.wxd && io.dpath.mem_waddr(0)
   io.imem.btb_update.bits.isReturn := mem_ctrl.jalr && io.dpath.mem_rs1_ra
+
+  io.imem.bht_update.valid := mem_reg_valid && mem_ctrl.branch && !take_pc_wb
+  io.imem.bht_update.bits.taken := io.dpath.mem_br_taken
+  io.imem.bht_update.bits.mispredict := io.dpath.mem_misprediction
+  io.imem.bht_update.bits.prediction.valid := mem_reg_btb_hit
+  io.imem.bht_update.bits.prediction.bits := mem_reg_btb_resp
+
+  io.imem.ras_update.valid := io.imem.btb_update.bits.isJump && !take_pc_wb
+  io.imem.ras_update.bits.isCall := mem_ctrl.wxd && io.dpath.mem_waddr(0)
+  io.imem.ras_update.bits.isReturn := mem_ctrl.jalr && io.dpath.mem_rs1_ra
+  io.imem.ras_update.bits.prediction.valid := mem_reg_btb_hit
+  io.imem.ras_update.bits.prediction.bits := mem_reg_btb_resp
+
   io.imem.req.valid := take_pc
 
   val bypassDst = Array(id_raddr1, id_raddr2)
@@ -595,7 +605,7 @@ class Control extends Module
      io.fpu.dec.ren3 && id_raddr3 === io.dpath.ex_waddr ||
      io.fpu.dec.wen  && id_waddr  === io.dpath.ex_waddr)
   val id_ex_hazard = ex_reg_valid && (data_hazard_ex && ex_cannot_bypass || fp_data_hazard_ex)
-    
+
   // stall for RAW/WAW hazards on PCRs, LB/LH, and mul/div in memory stage.
   val mem_mem_cmd_bh =
     if (params(FastLoadWord)) Bool(!params(FastLoadByte)) && mem_reg_slow_bypass

rocket/src/main/scala/dpath.scala

@@ -42,9 +42,9 @@ class Datapath extends Module
   val wb_reg_rs2 = Reg(Bits())
 
   // instruction decode stage
-  val id_inst = io.imem.resp.bits.data
+  val id_inst = io.imem.resp.bits.data(0).toBits; require(params(FetchWidth) == 1)
   val id_pc = io.imem.resp.bits.pc
- 
+
   class RegFile {
     private val rf = Mem(UInt(width = 64), 31)
     private val reads = collection.mutable.ArrayBuffer[(UInt,UInt)]()
@@ -276,7 +276,9 @@ class Datapath extends Module
         wb_reg_pc)).toUInt // PC_WB
   io.imem.btb_update.bits.pc := mem_reg_pc
   io.imem.btb_update.bits.target := io.imem.req.bits.pc
-  io.imem.btb_update.bits.returnAddr := mem_int_wdata
+  io.imem.btb_update.bits.br_pc := mem_reg_pc
+  io.imem.bht_update.bits.pc := mem_reg_pc
+  io.imem.ras_update.bits.returnAddr := mem_int_wdata
   
   // for hazard/bypass opportunity detection
   io.ctrl.ex_waddr := ex_reg_inst(11,7)

rocket/src/main/scala/icache.scala

@@ -1,5 +1,3 @@
-// See LICENSE for license details.
-
 package rocket
 
 import Chisel._
@@ -27,7 +25,8 @@ class FrontendReq extends CoreBundle {
 
 class FrontendResp extends CoreBundle {
   val pc = UInt(width = params(VAddrBits)+1)  // ID stage PC
-  val data = Bits(width = coreInstBits)
+  val data = Vec.fill(coreFetchWidth) (Bits(width = coreInstBits))
+  val mask = Bits(width = coreFetchWidth)
   val xcpt_ma = Bool()
   val xcpt_if = Bool()
 }
@@ -37,18 +36,20 @@ class CPUFrontendIO extends CoreBundle {
   val resp = Decoupled(new FrontendResp).flip
   val btb_resp = Valid(new BTBResp).flip
   val btb_update = Valid(new BTBUpdate)
+  val bht_update = Valid(new BHTUpdate)
+  val ras_update = Valid(new RASUpdate)
   val ptw = new TLBPTWIO().flip
   val invalidate = Bool(OUTPUT)
 }
 
-class Frontend extends FrontendModule
+class Frontend(btb_updates_out_of_order: Boolean = false) extends FrontendModule
 {
   val io = new Bundle {
     val cpu = new CPUFrontendIO().flip
     val mem = new UncachedTileLinkIO
   }
 
-  val btb = Module(new BTB)
+  val btb = Module(new BTB(btb_updates_out_of_order))
   val icache = Module(new ICache)
   val tlb = Module(new TLB(params(NITLBEntries)))
 
@@ -62,13 +63,14 @@ class Frontend extends FrontendModule
   val s2_xcpt_if = Reg(init=Bool(false))
 
   val msb = vaddrBits-1
+  val lsb = log2Up(coreFetchWidth*coreInstBytes)
   val btbTarget = Cat(btb.io.resp.bits.target(msb), btb.io.resp.bits.target)
-  val pcp4_0 = s1_pc + UInt(coreInstBytes)
+  val ntpc_0 = s1_pc + UInt(coreInstBytes*coreFetchWidth)
-  val pcp4 = Cat(s1_pc(msb) & pcp4_0(msb), pcp4_0(msb,0))
+  val ntpc = Cat(s1_pc(msb) & ntpc_0(msb), ntpc_0(msb,lsb), Bits(0,lsb)) // unsure
   val icmiss = s2_valid && !icache.io.resp.valid
-  val predicted_npc = Mux(btb.io.resp.bits.taken, btbTarget, pcp4)
+  val predicted_npc = Mux(btb.io.resp.bits.taken, btbTarget, ntpc)
   val npc = Mux(icmiss, s2_pc, predicted_npc).toUInt
-  val s0_same_block = !icmiss && !io.cpu.req.valid && !btb.io.resp.bits.taken && ((pcp4 & rowBytes) === (s1_pc & rowBytes))
+  val s0_same_block = !icmiss && !io.cpu.req.valid && !btb.io.resp.bits.taken && ((ntpc & rowBytes) === (s1_pc & rowBytes))
 
   val stall = io.cpu.resp.valid && !io.cpu.resp.ready
   when (!stall) {
@@ -90,7 +92,9 @@ class Frontend extends FrontendModule
 
   btb.io.req.valid := !stall && !icmiss
   btb.io.req.bits.addr := s1_pc & SInt(-coreInstBytes)
-  btb.io.update := io.cpu.btb_update
+  btb.io.btb_update := io.cpu.btb_update
+  btb.io.bht_update := io.cpu.bht_update
+  btb.io.ras_update := io.cpu.ras_update
   btb.io.invalidate := io.cpu.invalidate || io.cpu.ptw.invalidate
 
   tlb.io.ptw <> io.cpu.ptw
@@ -110,7 +114,17 @@ class Frontend extends FrontendModule
 
   io.cpu.resp.valid := s2_valid && (s2_xcpt_if || icache.io.resp.valid)
   io.cpu.resp.bits.pc := s2_pc & SInt(-coreInstBytes) // discard PC LSBs
-  io.cpu.resp.bits.data := icache.io.resp.bits.datablock >> (s2_pc(log2Up(rowBytes)-1,log2Up(coreInstBytes)) << log2Up(coreInstBits))
+
+
+  val fetch_data = icache.io.resp.bits.datablock >> (s2_pc(log2Up(rowBytes)-1,log2Up(coreFetchWidth*coreInstBytes)) << log2Up(coreFetchWidth*coreInstBits))
+  for (i <- 0 until coreFetchWidth) {
+    io.cpu.resp.bits.data(i) := fetch_data(i*coreInstBits+coreInstBits-1, i*coreInstBits)
+  }
+
+  val all_ones = UInt((1 << (coreFetchWidth+1))-1)
+  val msk_pc = if (coreFetchWidth == 1) all_ones else all_ones << s2_pc(log2Up(coreFetchWidth) -1+2,2)
+  io.cpu.resp.bits.mask := Mux(s2_btb_resp_valid, msk_pc & s2_btb_resp_bits.mask, msk_pc)
+
   io.cpu.resp.bits.xcpt_ma := s2_pc(log2Up(coreInstBytes)-1,0) != UInt(0)
   io.cpu.resp.bits.xcpt_if := s2_xcpt_if