| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -43,6 +43,11 @@ case class DCacheConfig(sets: Int, ways: Int,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val lrsc_cycles = 32 // ISA requires 16-insn LRSC sequences to succeed
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				abstract trait DCacheBundle extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  implicit val conf: DCacheConfig
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				abstract class ReplacementPolicy
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				{
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  def way: UInt
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -102,30 +107,22 @@ class MSHRReq(implicit conf: DCacheConfig) extends HellaCacheReq {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val tag_match = Bool()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val old_meta = new MetaData
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val way_en = Bits(width = conf.ways)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new MSHRReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class Replay(implicit conf: DCacheConfig) extends HellaCacheReq {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val sdq_id = UInt(width = log2Up(conf.nsdq))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new Replay().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class DataReadReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class DataReadReq(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val way_en = Bits(width = conf.ways)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val addr   = Bits(width = conf.untagbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new DataReadReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class DataWriteReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class DataWriteReq(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val way_en = Bits(width = conf.ways)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val addr   = Bits(width = conf.untagbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val wmask  = Bits(width = conf.wordsperrow)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val data   = Bits(width = conf.bitsperrow)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new DataWriteReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class InternalProbe(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends Probe {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -152,25 +149,19 @@ object MetaData {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    meta
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaData(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaData(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val state = UInt(width = conf.statebits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val tag = Bits(width = conf.tagbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new MetaData().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaReadReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaReadReq(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val addr  = UInt(width = conf.paddrbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new MetaReadReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaWriteReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MetaWriteReq(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val way_en = Bits(width = conf.ways)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val idx  = Bits(width = conf.idxbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val data = new MetaData()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new MetaWriteReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class MSHR(id: Int)(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends Module {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -420,7 +411,7 @@ class MSHRFile(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends M
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.mem_resp := memRespMux(io.mem_grant.bits.payload.client_xact_id)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val free_sdq = io.replay.fire() && isWrite(io.replay.bits.cmd)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.replay.bits.data := sdq(RegEn(replay_arb.io.out.bits.sdq_id, free_sdq))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.replay.bits.data := sdq(RegEnable(replay_arb.io.out.bits.sdq_id, free_sdq))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.replay <> replay_arb.io.out
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  when (io.replay.valid || sdq_enq) {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -597,7 +588,7 @@ class MetaDataArray(implicit conf: DCacheConfig, tl: TileLinkConfiguration) exte
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    val mask = Mux(rst, SInt(-1), io.write.bits.way_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val tag = tags(RegEn(io.read.bits.addr >> conf.offbits, io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val tag = tags(RegEnable(io.read.bits.addr >> conf.offbits, io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  for (w <- 0 until conf.ways) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    val m = tag(metabits*(w+1)-1, metabits*w)
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -624,7 +615,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      val wway_en = io.write.bits.way_en(w+conf.wordsperrow-1,w)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      val rway_en = io.read.bits.way_en(w+conf.wordsperrow-1,w)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      val resp = Vec.fill(conf.wordsperrow){Bits(width = conf.bitsperrow)}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      val r_raddr = RegEn(io.read.bits.addr, io.read.valid)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      val r_raddr = RegEnable(io.read.bits.addr, io.read.valid)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      for (p <- 0 until resp.size) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        val array = Mem(Bits(width=conf.bitsperrow), conf.sets*REFILL_CYCLES, seqRead = true)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        when (wway_en.orR && io.write.valid && io.write.bits.wmask(p)) {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -632,7 +623,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				          val mask = FillInterleaved(conf.encdatabits, wway_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				          array.write(waddr, data, mask)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        resp(p) := array(RegEn(raddr, rway_en.orR && io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        resp(p) := array(RegEnable(raddr, rway_en.orR && io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      for (dw <- 0 until conf.wordsperrow) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        val r = AVec(resp.map(_(conf.encdatabits*(dw+1)-1,conf.encdatabits*dw)))
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -649,7 +640,7 @@ class DataArray(implicit conf: DCacheConfig) extends Module {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      when (io.write.bits.way_en(w) && io.write.valid) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        array.write(waddr, io.write.bits.data, wmask)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      io.resp(w) := array(RegEn(raddr, io.read.bits.way_en(w) && io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				      io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -695,7 +686,7 @@ class AMOALU(implicit conf: DCacheConfig) extends Module {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.out := wmask & out | ~wmask & io.lhs
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class HellaCacheReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class HellaCacheReq(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val kill = Bool()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val typ  = Bits(width = 3)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val phys = Bool()
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -703,11 +694,9 @@ class HellaCacheReq(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val data = Bits(width = conf.databits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val tag  = Bits(width = conf.reqtagbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val cmd  = Bits(width = 4)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new HellaCacheReq().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class HellaCacheResp(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class HellaCacheResp(implicit val conf: DCacheConfig) extends DCacheBundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val nack = Bool() // comes 2 cycles after req.fire
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val replay = Bool()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val typ = Bits(width = 3)
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -717,8 +706,6 @@ class HellaCacheResp(implicit conf: DCacheConfig) extends Bundle {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val cmd  = Bits(width = 4)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val addr = UInt(width = conf.maxaddrbits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val store_data = Bits(width = conf.databits)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  override def clone = new HellaCacheResp().asInstanceOf[this.type]
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				class AlignmentExceptions extends Bundle {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -756,15 +743,15 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val mshrs = Module(new MSHRFile)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  io.cpu.req.ready := Bool(true)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_valid = Reg(update=io.cpu.req.fire(), reset=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_valid = Reg(updateData=io.cpu.req.fire(), resetData=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_req = Reg(io.cpu.req.bits.clone)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_valid_masked = s1_valid && !io.cpu.req.bits.kill
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_replay = RegReset(Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_clk_en = Reg(Bool())
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_valid = Reg(update=s1_valid_masked, reset=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_valid = Reg(updateData=s1_valid_masked, resetData=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_req = Reg(io.cpu.req.bits.clone)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_replay = Reg(update=s1_replay, reset=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_replay = Reg(updateData=s1_replay, resetData=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_recycle = Bool()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_valid_masked = Bool()
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -772,7 +759,7 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s3_req = Reg(io.cpu.req.bits.clone)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s3_way = Reg(Bits())
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_recycled = RegEn(s2_recycle, s1_clk_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_recycled = RegEnable(s2_recycle, s1_clk_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_read  = isRead(s1_req.cmd)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_write = isWrite(s1_req.cmd)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_readwrite = s1_read || s1_write || isPrefetch(s1_req.cmd)
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -872,9 +859,9 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  s1_clk_en := metaReadArb.io.out.valid
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_writeback = s1_clk_en && !s1_valid && !s1_replay
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_tag_match_way = RegEn(s1_tag_match_way, s1_clk_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_tag_match = s2_tag_match_way.orR
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEn(meta.io.resp(w).state, s1_clk_en)))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en)))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  // load-reserved/store-conditional
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -931,8 +918,8 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  // replacement policy
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val replacer = new RandomReplacement
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_replaced_way_en = UIntToOH(replacer.way)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_replaced_way_en = UIntToOH(RegEn(replacer.way, s1_clk_en))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEn(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  // miss handling
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  mshrs.io.req.valid := s2_valid_masked && !s2_hit && (isPrefetch(s2_req.cmd) || isRead(s2_req.cmd) || isWrite(s2_req.cmd))
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -993,8 +980,8 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  FIFOedLogicalNetworkIOWrapper(wb.io.release_data) <> io.mem.release.data
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  // store->load bypassing
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s4_valid = Reg(update=s3_valid, reset=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s4_req = RegEn(s3_req, s3_valid && metaReadArb.io.out.valid)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s4_valid = Reg(updateData=s3_valid, resetData=Bool(false))
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s4_req = RegEnable(s3_req, s3_valid && metaReadArb.io.out.valid)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val bypasses = List(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ((s2_valid_masked || s2_replay) && !s2_sc_fail, s2_req, amoalu.io.out),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    (s3_valid, s3_req, s3_req.data),
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -1022,7 +1009,7 @@ class HellaCache(implicit conf: DCacheConfig, tl: TileLinkConfiguration) extends
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  // nack it like it's hot
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s1_nack = dtlb.io.req.valid && dtlb.io.resp.miss ||
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                s1_req.addr(indexmsb,indexlsb) === prober.io.meta_write.bits.idx && !prober.io.req.ready
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_nack_hit = RegEn(s1_nack, s1_valid || s1_replay)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_nack_hit = RegEnable(s1_nack, s1_valid || s1_replay)
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  when (s2_nack_hit) { mshrs.io.req.valid := Bool(false) }
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_nack_victim = s2_hit && mshrs.io.secondary_miss
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				  val s2_nack_miss = !s2_hit && !mshrs.io.req.ready
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				 
 |