|  |  |  | @@ -43,6 +43,11 @@ case class DCacheConfig(sets: Int, ways: Int, | 
		
	
		
			
				|  |  |  |  |   val lrsc_cycles = 32 // ISA requires 16-insn LRSC sequences to succeed | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | abstract trait DCacheBundle extends Bundle { | 
		
	
		
			
				|  |  |  |  |   implicit val conf: DCacheConfig | 
		
	
		
			
				|  |  |  |  |   override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | abstract class ReplacementPolicy | 
		
	
		
			
				|  |  |  |  | { | 
		
	
		
			
				|  |  |  |  |   def way: UInt | 
		
	
	
		
			
				
					
					|  |  |  | @@ -102,30 +107,22 @@ class MSHRReq(implicit conf: DCacheConfig) extends HellaCacheReq { | 
		
	
		
			
				|  |  |  |  |   val tag_match = Bool() | 
		
	
		
			
				|  |  |  |  |   val old_meta = new MetaData | 
		
	
		
			
				|  |  |  |  |   val way_en = Bits(width = conf.ways) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new MSHRReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class Replay(implicit conf: DCacheConfig) extends HellaCacheReq { | 
		
	
		
			
				|  |  |  |  |   val sdq_id = UInt(width = log2Up(conf.nsdq)) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new Replay().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class DataReadReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class DataReadReq(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val way_en = Bits(width = conf.ways) | 
		
	
		
			
				|  |  |  |  |   val addr   = Bits(width = conf.untagbits) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new DataReadReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class DataWriteReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class DataWriteReq(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val way_en = Bits(width = conf.ways) | 
		
	
		
			
				|  |  |  |  |   val addr   = Bits(width = conf.untagbits) | 
		
	
		
			
				|  |  |  |  |   val wmask  = Bits(width = conf.wordsperrow) | 
		
	
		
			
				|  |  |  |  |   val data   = Bits(width = conf.bitsperrow) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new DataWriteReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class InternalProbe(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends Probe { | 
		
	
	
		
			
				
					
					|  |  |  | @@ -152,25 +149,19 @@ object MetaData { | 
		
	
		
			
				|  |  |  |  |     meta | 
		
	
		
			
				|  |  |  |  |   } | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  | class MetaData(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class MetaData(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val state = UInt(width = conf.statebits) | 
		
	
		
			
				|  |  |  |  |   val tag = Bits(width = conf.tagbits) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new MetaData().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class MetaReadReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class MetaReadReq(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val addr  = UInt(width = conf.paddrbits) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new MetaReadReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class MetaWriteReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class MetaWriteReq(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val way_en = Bits(width = conf.ways) | 
		
	
		
			
				|  |  |  |  |   val idx  = Bits(width = conf.idxbits) | 
		
	
		
			
				|  |  |  |  |   val data = new MetaData() | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new MetaWriteReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class MSHR(id: Int)(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends Module { | 
		
	
	
		
			
				
					
					|  |  |  | @@ -420,7 +411,7 @@ class MSHRFile(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends M | 
		
	
		
			
				|  |  |  |  |   io.mem_resp := memRespMux(io.mem_grant.bits.payload.client_xact_id) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   val free_sdq = io.replay.fire() && isWrite(io.replay.bits.cmd) | 
		
	
		
			
				|  |  |  |  |   io.replay.bits.data := sdq(RegEn(replay_arb.io.out.bits.sdq_id, free_sdq)) | 
		
	
		
			
				|  |  |  |  |   io.replay.bits.data := sdq(RegEnable(replay_arb.io.out.bits.sdq_id, free_sdq)) | 
		
	
		
			
				|  |  |  |  |   io.replay <> replay_arb.io.out | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   when (io.replay.valid || sdq_enq) { | 
		
	
	
		
			
				
					
					|  |  |  | @@ -597,7 +588,7 @@ class MetaDataArray(implicit conf: DCacheConfig, tl: TileLinkConfiguration) exte | 
		
	
		
			
				|  |  |  |  |     val mask = Mux(rst, SInt(-1), io.write.bits.way_en) | 
		
	
		
			
				|  |  |  |  |     tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) | 
		
	
		
			
				|  |  |  |  |   } | 
		
	
		
			
				|  |  |  |  |   val tag = tags(RegEn(io.read.bits.addr >> conf.offbits, io.read.valid)) | 
		
	
		
			
				|  |  |  |  |   val tag = tags(RegEnable(io.read.bits.addr >> conf.offbits, io.read.valid)) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   for (w <- 0 until conf.ways) { | 
		
	
		
			
				|  |  |  |  |     val m = tag(metabits*(w+1)-1, metabits*w) | 
		
	
	
		
			
				
					
					|  |  |  | @@ -624,7 +615,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module { | 
		
	
		
			
				|  |  |  |  |       val wway_en = io.write.bits.way_en(w+conf.wordsperrow-1,w) | 
		
	
		
			
				|  |  |  |  |       val rway_en = io.read.bits.way_en(w+conf.wordsperrow-1,w) | 
		
	
		
			
				|  |  |  |  |       val resp = Vec.fill(conf.wordsperrow){Bits(width = conf.bitsperrow)} | 
		
	
		
			
				|  |  |  |  |       val r_raddr = RegEn(io.read.bits.addr, io.read.valid) | 
		
	
		
			
				|  |  |  |  |       val r_raddr = RegEnable(io.read.bits.addr, io.read.valid) | 
		
	
		
			
				|  |  |  |  |       for (p <- 0 until resp.size) { | 
		
	
		
			
				|  |  |  |  |         val array = Mem(Bits(width=conf.bitsperrow), conf.sets*REFILL_CYCLES, seqRead = true) | 
		
	
		
			
				|  |  |  |  |         when (wway_en.orR && io.write.valid && io.write.bits.wmask(p)) { | 
		
	
	
		
			
				
					
					|  |  |  | @@ -632,7 +623,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module { | 
		
	
		
			
				|  |  |  |  |           val mask = FillInterleaved(conf.encdatabits, wway_en) | 
		
	
		
			
				|  |  |  |  |           array.write(waddr, data, mask) | 
		
	
		
			
				|  |  |  |  |         } | 
		
	
		
			
				|  |  |  |  |         resp(p) := array(RegEn(raddr, rway_en.orR && io.read.valid)) | 
		
	
		
			
				|  |  |  |  |         resp(p) := array(RegEnable(raddr, rway_en.orR && io.read.valid)) | 
		
	
		
			
				|  |  |  |  |       } | 
		
	
		
			
				|  |  |  |  |       for (dw <- 0 until conf.wordsperrow) { | 
		
	
		
			
				|  |  |  |  |         val r = AVec(resp.map(_(conf.encdatabits*(dw+1)-1,conf.encdatabits*dw))) | 
		
	
	
		
			
				
					
					|  |  |  | @@ -649,7 +640,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module { | 
		
	
		
			
				|  |  |  |  |       when (io.write.bits.way_en(w) && io.write.valid) { | 
		
	
		
			
				|  |  |  |  |         array.write(waddr, io.write.bits.data, wmask) | 
		
	
		
			
				|  |  |  |  |       } | 
		
	
		
			
				|  |  |  |  |       io.resp(w) := array(RegEn(raddr, io.read.bits.way_en(w) && io.read.valid)) | 
		
	
		
			
				|  |  |  |  |       io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) | 
		
	
		
			
				|  |  |  |  |     } | 
		
	
		
			
				|  |  |  |  |   } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
	
		
			
				
					
					|  |  |  | @@ -695,7 +686,7 @@ class AMOALU(implicit conf: DCacheConfig) extends Module { | 
		
	
		
			
				|  |  |  |  |   io.out := wmask & out | ~wmask & io.lhs | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class HellaCacheReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class HellaCacheReq(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val kill = Bool() | 
		
	
		
			
				|  |  |  |  |   val typ  = Bits(width = 3) | 
		
	
		
			
				|  |  |  |  |   val phys = Bool() | 
		
	
	
		
			
				
					
					|  |  |  | @@ -703,11 +694,9 @@ class HellaCacheReq(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  |   val data = Bits(width = conf.databits) | 
		
	
		
			
				|  |  |  |  |   val tag  = Bits(width = conf.reqtagbits) | 
		
	
		
			
				|  |  |  |  |   val cmd  = Bits(width = 4) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new HellaCacheReq().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class HellaCacheResp(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  | class HellaCacheResp(implicit val conf: DCacheConfig) extends DCacheBundle { | 
		
	
		
			
				|  |  |  |  |   val nack = Bool() // comes 2 cycles after req.fire | 
		
	
		
			
				|  |  |  |  |   val replay = Bool() | 
		
	
		
			
				|  |  |  |  |   val typ = Bits(width = 3) | 
		
	
	
		
			
				
					
					|  |  |  | @@ -717,8 +706,6 @@ class HellaCacheResp(implicit conf: DCacheConfig) extends Bundle { | 
		
	
		
			
				|  |  |  |  |   val cmd  = Bits(width = 4) | 
		
	
		
			
				|  |  |  |  |   val addr = UInt(width = conf.maxaddrbits) | 
		
	
		
			
				|  |  |  |  |   val store_data = Bits(width = conf.databits) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   override def clone = new HellaCacheResp().asInstanceOf[this.type] | 
		
	
		
			
				|  |  |  |  | } | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  | class AlignmentExceptions extends Bundle { | 
		
	
	
		
			
				
					
					|  |  |  | @@ -756,15 +743,15 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   val mshrs = Module(new MSHRFile) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   io.cpu.req.ready := Bool(true) | 
		
	
		
			
				|  |  |  |  |   val s1_valid = Reg(update=io.cpu.req.fire(), reset=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s1_valid = Reg(updateData=io.cpu.req.fire(), resetData=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s1_req = Reg(io.cpu.req.bits.clone) | 
		
	
		
			
				|  |  |  |  |   val s1_valid_masked = s1_valid && !io.cpu.req.bits.kill | 
		
	
		
			
				|  |  |  |  |   val s1_replay = RegReset(Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s1_clk_en = Reg(Bool()) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   val s2_valid = Reg(update=s1_valid_masked, reset=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s2_valid = Reg(updateData=s1_valid_masked, resetData=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s2_req = Reg(io.cpu.req.bits.clone) | 
		
	
		
			
				|  |  |  |  |   val s2_replay = Reg(update=s1_replay, reset=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s2_replay = Reg(updateData=s1_replay, resetData=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s2_recycle = Bool() | 
		
	
		
			
				|  |  |  |  |   val s2_valid_masked = Bool() | 
		
	
		
			
				|  |  |  |  |  | 
		
	
	
		
			
				
					
					|  |  |  | @@ -772,7 +759,7 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   val s3_req = Reg(io.cpu.req.bits.clone) | 
		
	
		
			
				|  |  |  |  |   val s3_way = Reg(Bits()) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   val s1_recycled = RegEn(s2_recycle, s1_clk_en) | 
		
	
		
			
				|  |  |  |  |   val s1_recycled = RegEnable(s2_recycle, s1_clk_en) | 
		
	
		
			
				|  |  |  |  |   val s1_read  = isRead(s1_req.cmd) | 
		
	
		
			
				|  |  |  |  |   val s1_write = isWrite(s1_req.cmd) | 
		
	
		
			
				|  |  |  |  |   val s1_readwrite = s1_read || s1_write || isPrefetch(s1_req.cmd) | 
		
	
	
		
			
				
					
					|  |  |  | @@ -872,9 +859,9 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits | 
		
	
		
			
				|  |  |  |  |   s1_clk_en := metaReadArb.io.out.valid | 
		
	
		
			
				|  |  |  |  |   val s1_writeback = s1_clk_en && !s1_valid && !s1_replay | 
		
	
		
			
				|  |  |  |  |   val s2_tag_match_way = RegEn(s1_tag_match_way, s1_clk_en) | 
		
	
		
			
				|  |  |  |  |   val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) | 
		
	
		
			
				|  |  |  |  |   val s2_tag_match = s2_tag_match_way.orR | 
		
	
		
			
				|  |  |  |  |   val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEn(meta.io.resp(w).state, s1_clk_en))) | 
		
	
		
			
				|  |  |  |  |   val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) | 
		
	
		
			
				|  |  |  |  |   val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   // load-reserved/store-conditional | 
		
	
	
		
			
				
					
					|  |  |  | @@ -931,8 +918,8 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   // replacement policy | 
		
	
		
			
				|  |  |  |  |   val replacer = new RandomReplacement | 
		
	
		
			
				|  |  |  |  |   val s1_replaced_way_en = UIntToOH(replacer.way) | 
		
	
		
			
				|  |  |  |  |   val s2_replaced_way_en = UIntToOH(RegEn(replacer.way, s1_clk_en)) | 
		
	
		
			
				|  |  |  |  |   val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEn(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) | 
		
	
		
			
				|  |  |  |  |   val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) | 
		
	
		
			
				|  |  |  |  |   val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   // miss handling | 
		
	
		
			
				|  |  |  |  |   mshrs.io.req.valid := s2_valid_masked && !s2_hit && (isPrefetch(s2_req.cmd) || isRead(s2_req.cmd) || isWrite(s2_req.cmd)) | 
		
	
	
		
			
				
					
					|  |  |  | @@ -993,8 +980,8 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   FIFOedLogicalNetworkIOWrapper(wb.io.release_data) <> io.mem.release.data | 
		
	
		
			
				|  |  |  |  |  | 
		
	
		
			
				|  |  |  |  |   // store->load bypassing | 
		
	
		
			
				|  |  |  |  |   val s4_valid = Reg(update=s3_valid, reset=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s4_req = RegEn(s3_req, s3_valid && metaReadArb.io.out.valid) | 
		
	
		
			
				|  |  |  |  |   val s4_valid = Reg(updateData=s3_valid, resetData=Bool(false)) | 
		
	
		
			
				|  |  |  |  |   val s4_req = RegEnable(s3_req, s3_valid && metaReadArb.io.out.valid) | 
		
	
		
			
				|  |  |  |  |   val bypasses = List( | 
		
	
		
			
				|  |  |  |  |     ((s2_valid_masked || s2_replay) && !s2_sc_fail, s2_req, amoalu.io.out), | 
		
	
		
			
				|  |  |  |  |     (s3_valid, s3_req, s3_req.data), | 
		
	
	
		
			
				
					
					|  |  |  | @@ -1022,7 +1009,7 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends | 
		
	
		
			
				|  |  |  |  |   // nack it like it's hot | 
		
	
		
			
				|  |  |  |  |   val s1_nack = dtlb.io.req.valid && dtlb.io.resp.miss || | 
		
	
		
			
				|  |  |  |  |                 s1_req.addr(indexmsb,indexlsb) === prober.io.meta_write.bits.idx && !prober.io.req.ready | 
		
	
		
			
				|  |  |  |  |   val s2_nack_hit = RegEn(s1_nack, s1_valid || s1_replay) | 
		
	
		
			
				|  |  |  |  |   val s2_nack_hit = RegEnable(s1_nack, s1_valid || s1_replay) | 
		
	
		
			
				|  |  |  |  |   when (s2_nack_hit) { mshrs.io.req.valid := Bool(false) } | 
		
	
		
			
				|  |  |  |  |   val s2_nack_victim = s2_hit && mshrs.io.secondary_miss | 
		
	
		
			
				|  |  |  |  |   val s2_nack_miss = !s2_hit && !mshrs.io.req.ready | 
		
	
	
		
			
				
					
					|  |  |  |   |