more itlb/dtlb/ptw fixes

rocket/src/main/scala/cpu.scala

@@ -70,7 +70,7 @@ class rocketProc extends Component
   ctrl.io.imem.resp_val   := io.imem.resp_val;
   dpath.io.imem.resp_data := io.imem.resp_data;
   ctrl.io.xcpt_itlb       := itlb.io.cpu.exception;
-  ctrl.io.itlb_miss       := itlb.io.cpu.resp_miss;
+//   ctrl.io.itlb_miss       := itlb.io.cpu.resp_miss;
   io.imem.itlb_miss       := itlb.io.cpu.resp_miss;
 
   
@@ -83,7 +83,9 @@ class rocketProc extends Component
   dtlb.io.cpu.req_vpn     := dpath.io.dmem.req_addr(VADDR_BITS-1,PGIDX_BITS);
   ctrl.io.xcpt_dtlb_ld    := dtlb.io.cpu.xcpt_ld; 
   ctrl.io.xcpt_dtlb_st    := dtlb.io.cpu.xcpt_st; 
+  ctrl.io.dtlb_busy       := dtlb.io.cpu.resp_busy;
   ctrl.io.dtlb_miss       := dtlb.io.cpu.resp_miss;
+//   io.dmem.dtlb_miss       := dtlb.io.cpu.resp_miss;
   
   // connect page table walker to TLBs, page table base register (from PCR)
   // and D$ arbiter (selects between requests from pipeline and PTW, PTW has priority)
@@ -98,6 +100,9 @@ class rocketProc extends Component
   arb.io.cpu.req_val      := ctrl.io.dmem.req_val;
   arb.io.cpu.req_cmd      := ctrl.io.dmem.req_cmd;
   arb.io.cpu.req_type     := ctrl.io.dmem.req_type;
+//   arb.io.cpu.dtlb_busy    := dtlb.io.cpu.resp_busy;
+  arb.io.cpu.dtlb_miss    := dtlb.io.cpu.resp_miss;
+
 //   arb.io.cpu.req_addr     := dtlb.io.cpu.resp_addr;
   arb.io.cpu.req_idx      := dpath.io.dmem.req_addr(PGIDX_BITS-1,0);
   arb.io.cpu.req_ppn      := dtlb.io.cpu.resp_ppn;

rocket/src/main/scala/ctrl.scala

@@ -67,8 +67,9 @@ class ioCtrlAll extends Bundle()
   val imem    = new ioImem(List("req_val", "req_rdy", "resp_val")).flip();
   val dmem    = new ioDmem(List("req_val", "req_rdy", "req_cmd", "req_type", "resp_miss")).flip();
   val host    = new ioHost(List("start"));
+  val dtlb_busy = Bool('input);
   val dtlb_miss = Bool('input);
-  val itlb_miss = Bool('input);
+//   val itlb_miss = Bool('input);
   val xcpt_dtlb_ld = Bool('input);
   val xcpt_dtlb_st = Bool('input);
   val xcpt_itlb = Bool('input);
@@ -497,7 +498,7 @@ class rocketCtrl extends Component
       id_ren1 &  id_stall_raddr1 |
       (id_sel_wa === WA_RD) & id_stall_waddr |
       (id_sel_wa === WA_RA) & id_stall_ra |
-      id_mem_val & ~io.dmem.req_rdy |
+      id_mem_val & (~io.dmem.req_rdy | io.dtlb_busy) |
       id_sync & ~io.dmem.req_rdy |
       id_console_out_val & ~io.console.rdy |
       id_div_val & ~io.dpath.div_rdy |
@@ -513,7 +514,8 @@ class rocketCtrl extends Component
 
   io.dpath.stalld   := ctrl_stalld.toBool;
 
-  io.dpath.killf    := take_pc | io.itlb_miss | ~io.imem.resp_val;
+//   io.dpath.killf    := take_pc | io.itlb_miss | ~io.imem.resp_val;
+  io.dpath.killf    := take_pc | ~io.imem.resp_val;
   io.dpath.killd    := ctrl_killd.toBool;
   io.dpath.killx    := kill_ex.toBool;
   io.dpath.killm    := kill_mem.toBool;

rocket/src/main/scala/dcache.scala

@@ -7,6 +7,8 @@ import scala.math._;
 
 // interface between D$ and processor/DTLB
 class ioDmem(view: List[String] = null) extends Bundle(view) {
+//   val dtlb_busy  = Bool('input);
+  val dtlb_miss  = Bool('input);
   val req_val   = Bool('input);
   val req_rdy   = Bool('output);
   val req_cmd   = Bits(4, 'input);
@@ -155,6 +157,8 @@ class rocketDCacheDM_flush(lines: Int) extends Component {
   dcache.io.cpu.req_tag   := Mux(flushing, r_cpu_req_tag, io.cpu.req_tag);
   dcache.io.cpu.req_type  := io.cpu.req_type;
   dcache.io.cpu.req_data  ^^ io.cpu.req_data;
+//   dcache.io.cpu.dtlb_busy := io.cpu.dtlb_busy;
+  dcache.io.cpu.dtlb_miss := io.cpu.dtlb_miss;
   dcache.io.mem           ^^ io.mem;
 
   io.cpu.req_rdy   := dcache.io.cpu.req_rdy && !flush_waiting;
@@ -210,19 +214,21 @@ class rocketDCacheDM(lines: Int) extends Component {
     r_cpu_req_tag   <== io.cpu.req_tag;
   }
   
-  when ((state === s_ready) && r_cpu_req_val) {
+  when ((state === s_ready) && r_cpu_req_val && !io.cpu.dtlb_miss) {
     r_cpu_req_ppn <== io.cpu.req_ppn;
   }
   when (io.cpu.req_rdy) {
     r_cpu_req_val <== io.cpu.req_val; 
   }
-  when (state === s_resolve_miss) {
+  otherwise {
     r_cpu_req_val <== Bool(false);
-    when (r_req_load) {
+  }
+  when (((state === s_resolve_miss) && r_req_load) || (state === s_replay_load)) {
     r_cpu_resp_val <== Bool(true);
   }
-  }
+  otherwise {
     r_cpu_resp_val <== Bool(false);
+  }
   
   // refill counter
   val rr_count = Reg(resetVal = UFix(0,2));
@@ -244,7 +250,7 @@ class rocketDCacheDM(lines: Int) extends Component {
   tag_array.io.d    := r_cpu_req_ppn;
   tag_array.io.we   := tag_we;
   tag_array.io.bweb := ~Bits(0,tagbits);
-  tag_array.io.ce   := (state === s_ready) && io.cpu.req_val;
+  tag_array.io.ce   := (state === s_ready) || (state === s_start_writeback) || (state === s_writeback);
   val tag_rdata      = tag_array.io.q;
 
   // valid bit array
@@ -256,8 +262,9 @@ class rocketDCacheDM(lines: Int) extends Component {
   when (tag_we && r_req_flush) {
     vb_array <== vb_array.bitSet(r_cpu_req_idx(PGIDX_BITS-1,offsetbits).toUFix, UFix(0,1));
   }
-
+  val vb_rdata = Reg(vb_array(tag_addr).toBool);
-  val tag_valid = Reg(vb_array(tag_addr)).toBool;
+  val tag_valid = r_cpu_req_val && vb_rdata;
+//   val tag_valid = Reg(vb_array(tag_addr)).toBool;
   val tag_match = (tag_rdata === io.cpu.req_ppn);
   val addr_match    = (r_cpu_req_idx(PGIDX_BITS-1,offsetbits) === p_store_idx(PGIDX_BITS-1,offsetbits));
   val ldst_conflict = r_cpu_req_val && r_req_load && p_store_valid && addr_match;
@@ -265,7 +272,7 @@ class rocketDCacheDM(lines: Int) extends Component {
   // write the pending store data when the cache is idle, when the next command isn't a load
   // or when there's a load to the same address (in which case there's a 2 cycle delay:
   // once cycle to write the store data and another to read the data back) 
-  val drain_store = p_store_valid && (!io.cpu.req_val || req_store || ldst_conflict);
+  val drain_store = !io.cpu.dtlb_miss && p_store_valid && (!io.cpu.req_val || req_store || ldst_conflict);
 
   // write pending store data from a store which missed
   // after the cache line refill has completed
@@ -282,6 +289,11 @@ class rocketDCacheDM(lines: Int) extends Component {
     p_store_type  <== io.cpu.req_type;
     p_store_valid <== Bool(true);  
   }
+  // cancel store if there's a DTLB miss
+  when (r_cpu_req_val && r_req_store && io.cpu.dtlb_miss)
+  {
+    p_store_valid <== Bool(false);
+  }
   when (drain_store)  {
     p_store_valid <== Bool(false);
     db_array <== db_array.bitSet(p_store_idx(PGIDX_BITS-1,offsetbits).toUFix, UFix(1,1));
@@ -321,12 +333,12 @@ class rocketDCacheDM(lines: Int) extends Component {
   // signal a load miss when the data isn't present in the cache and when it's in the pending store data register
   // (causes the cache to block for 2 cycles and the load instruction is replayed)
   val hit  = tag_valid && tag_match;
-  val load_miss = (state === s_ready) && r_cpu_req_val && r_req_load && (!hit || (p_store_valid && addr_match));
+  val load_miss = !io.cpu.dtlb_miss && (state === s_ready) && r_cpu_req_val && r_req_load && (!hit || (p_store_valid && addr_match));
 
   // output signals
   // busy when there's a load to the same address as a pending store, or on a cache miss, or when executing a flush
-  io.cpu.req_rdy   := (state === s_ready) && !ldst_conflict && (!r_cpu_req_val || (hit && !r_req_flush));
+  io.cpu.req_rdy   := !io.cpu.dtlb_miss && (state === s_ready) && !ldst_conflict && (!r_cpu_req_val || (hit && !r_req_flush));
-  io.cpu.resp_val  := ((state === s_ready) && r_cpu_req_val && r_req_load && hit && !(p_store_valid && addr_match)) || 
+  io.cpu.resp_val  := !io.cpu.dtlb_miss && ((state === s_ready) &&  hit && r_req_load && !(p_store_valid && addr_match)) || 
                       ((state === s_resolve_miss) && r_req_flush) ||
                       r_cpu_resp_val;
                       
@@ -351,6 +363,9 @@ class rocketDCacheDM(lines: Int) extends Component {
       state <== s_ready;
     }
     is (s_ready) {
+      when (io.cpu.dtlb_miss) {
+        state <== s_ready;
+      }
       when (ldst_conflict) {
         state <== s_replay_load;
       }

rocket/src/main/scala/dtlb.scala

@@ -20,6 +20,7 @@ class ioDTLB_CPU(view: List[String] = null) extends Bundle(view)
   val req_asid = Bits(ASID_BITS, 'input);
   val req_vpn  = UFix(VPN_BITS, 'input);
   // lookup responses
+  val resp_busy = Bool('output);
   val resp_miss = Bool('output);
 //   val resp_val = Bool('output);
   val resp_ppn = UFix(PPN_BITS, 'output);
@@ -62,6 +63,7 @@ class rocketDTLB(entries: Int) extends Component
   
   val req_load  = (r_cpu_req_cmd === M_XRD);
   val req_store = (r_cpu_req_cmd === M_XWR);
+  val req_flush = (r_cpu_req_cmd === M_FLA);
 //   val req_amo   = io.cpu.req_cmd(3).toBool;
   
   val lookup_tag = Cat(r_cpu_req_asid, r_cpu_req_vpn);
@@ -117,8 +119,8 @@ class rocketDTLB(entries: Int) extends Component
   
   val repl_waddr = Mux(invalid_entry, ie_addr, repl_count).toUFix;
   
-  val lookup_hit  = (state === s_ready) && r_cpu_req_val && tag_hit;
+  val lookup_hit  = (state === s_ready) && r_cpu_req_val && !req_flush && tag_hit;
-  val lookup_miss = (state === s_ready) && r_cpu_req_val && !tag_hit;
+  val lookup_miss = (state === s_ready) && r_cpu_req_val && !req_flush && !tag_hit;
   val tlb_hit  = status_vm && lookup_hit;
   val tlb_miss = status_vm && lookup_miss;
   
@@ -143,9 +145,12 @@ class rocketDTLB(entries: Int) extends Component
     ((status_s && !sw_array(tag_hit_addr).toBool) ||
      (status_u && !uw_array(tag_hit_addr).toBool));
 
-  io.cpu.req_rdy   := (state === s_ready);
+  io.cpu.req_rdy   := Mux(status_vm, (state === s_ready) && !tlb_miss, Bool(true));
+  io.cpu.resp_busy := tlb_miss || (state != s_ready);
   io.cpu.resp_miss := tlb_miss;
-  io.cpu.resp_ppn := Mux(status_vm, tag_ram(tag_hit_addr), r_cpu_req_vpn(PPN_BITS-1,0)).toUFix;
+  io.cpu.resp_ppn  := 
+    Mux(status_vm, Mux(req_flush, Bits(0,PPN_BITS), tag_ram(tag_hit_addr)), 
+      r_cpu_req_vpn(PPN_BITS-1,0)).toUFix;
   
   io.ptw.req_val := (state === s_request);
   io.ptw.req_vpn := r_refill_tag(VPN_BITS-1,0);

rocket/src/main/scala/icache.scala

@@ -81,7 +81,7 @@ class rocketICacheDM(lines: Int) extends Component {
   when (io.cpu.req_val && io.cpu.req_rdy) {
     r_cpu_req_idx   <== io.cpu.req_idx;
   }
-  when (state === s_ready) {
+  when (state === s_ready && r_cpu_req_val && !io.cpu.itlb_miss) {
     r_cpu_req_ppn <== io.cpu.req_ppn;
   }
   when (io.cpu.req_rdy) {
@@ -140,7 +140,9 @@ class rocketICacheDM(lines: Int) extends Component {
   data_array.io.d    := io.mem.resp_data;
   data_array.io.we   := io.mem.resp_val;
   data_array.io.bweb := ~Bits(0,128);
-  data_array.io.ce   := Bool(true); // FIXME
+//   data_array.io.ce   := Bool(true); // FIXME
+  data_array.io.ce   := (io.cpu.req_rdy && io.cpu.req_val) || (state === s_resolve_miss);
+    
   val data_array_rdata = data_array.io.q;
    
   // output signals
@@ -162,7 +164,12 @@ class rocketICacheDM(lines: Int) extends Component {
       state <== s_ready;
     }
     is (s_ready) {
-      when (!io.cpu.itlb_miss && r_cpu_req_val && !(tag_valid && tag_match)) { state <== s_request; }
+      when (io.cpu.itlb_miss) {
+        state <== s_ready;
+      }
+      when (r_cpu_req_val && !(tag_valid && tag_match)) {
+        state <== s_request;
+      }
     }
     is (s_request)
     {

rocket/src/main/scala/ptw.scala

@@ -28,6 +28,8 @@ class rocketDmemArbiter extends Component
   io.mem.req_ppn  := Mux(r_ptw_req_val, r_ptw_req_ppn, io.cpu.req_ppn);
   io.mem.req_data := io.cpu.req_data;
   io.mem.req_tag  := Mux(io.ptw.req_val, Bits(0,5), io.cpu.req_tag);
+//   io.mem.dtlb_busy := io.cpu.dtlb_busy;
+  io.mem.dtlb_miss := io.cpu.dtlb_miss;
   
   io.ptw.req_rdy   := io.mem.req_rdy;
   io.cpu.req_rdy   := io.mem.req_rdy && !io.ptw.req_val;