diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e7f0ce8a..6651056f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,32 +2,26 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] -case object WordBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +case object AmoAluOperandBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { - val paddrBits = params(PAddrBits) - val vaddrBits = params(VAddrBits) - val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) val blockOffBits = params(BlockOffBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits - val tagBits = paddrBits - untagBits + val tagBits = params(PAddrBits) - untagBits val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 - val wordBits = params(WordBits) - val wordBytes = wordBits/8 - val wordOffBits = log2Up(wordBytes) val rowBits = params(RowBits) - val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) } @@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters { abstract class CacheBundle extends Bundle with CacheParameters abstract class CacheModule extends Module with CacheParameters +class StoreGen(typ: Bits, addr: Bits, dat: Bits) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + lazy val wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { + val t = new StoreGen(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class AMOALU extends CacheModule { + val operandBits = params(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = new StoreGen(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.write.ready := !rst } -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters { +abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 val refillCyclesPerBeat = params(TLDataBits)/rowBits val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + require(refillCyclesPerBeat == 1) } -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { + def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { + val (cnt, cnt_done) = + Counter(inc && data.hasMultibeatData(), tlDataBeats) + val done = (inc && !data.hasMultibeatData()) || cnt_done + (cnt, done) + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload)._2 + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { + connectDataBeatCounter(in.fire(), in.bits) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { + connectDataBeatCounter(in.valid, in.bits)._2 + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } +trait HasL2Data extends HasTileLinkData { + def hasData(dummy: Int = 0) = Bool(true) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) +} + object L2Metadata { def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata @@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { +class L2DataReadReq extends L2HellaCacheBundle + with HasCacheBlockAddress + with HasTileLinkBeatId + with HasL2Id { val way_en = Bits(width = nWays) - val addr = Bits(width = tlAddrBits) } -class L2DataWriteReq extends L2DataReadReq { +class L2DataWriteReq extends L2DataReadReq + with HasL2Data { val wmask = Bits(width = tlWriteMaskBits) - val data = Bits(width = tlDataBits) } -class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { - val data = Bits(width = tlDataBits) -} +class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data trait HasL2DataReadIO extends L2HellaCacheBundle { val read = Decoupled(new L2DataReadReq) @@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule { io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) io.resp.bits.data := array(reg_raddr) io.read.ready := !io.write.valid io.write.ready := Bool(true) @@ -261,10 +353,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { + def doOutputArbitration[T <: Data]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } @@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + doOutputArbitration( + io.inner.grant, + trackerList.map(_.io.inner.grant), + tlDataBeats, + (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer @@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr = UInt(width = tlAddrBits) + val addr_block = UInt(width = tlBlockAddrBits) val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } @@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val pending_probes = Reg(init = co.dir.flush) val curr_p_id = co.dir.next(pending_probes) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) + val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) - io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, - UInt(trackerId), - xact_data(outer_data_write_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), { case TLId => outerId }) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.wb.req.ready := Bool(false) @@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str is(s_idle) { io.wb.req.ready := Bool(true) when(io.wb.req.valid) { - xact_addr := io.wb.req.bits.addr + xact_addr_block := io.wb.req.bits.addr_block xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id @@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str when(io.inner.release.valid) { xact_coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - xact_data(crel_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_read_done) { state := s_data_resp } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_write } } is(s_outer_write) { io.outer.acquire.valid := Bool(true) - when(outer_data_write_done) { + when(macq_data_done) { io.wb.resp.valid := Bool(true) state := s_idle } @@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val wb = new L2WritebackIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - - def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { - //TODO apply acq's write mask - Mux(co.messageHasData(acq), old_data, new_data) - } + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cack = io.inner.finish.bits + val mgnt = io.outer.grant.bits } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) - val collect_inner_data = Reg(init=Bool(false)) - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (local_data_cnt, local_data_done) = - Counter(io.data.write.fire(), tlDataBeats) + val collect_crel_data = Reg(init=Bool(false)) + val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) - when(collect_inner_data) { + when(collect_crel_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data + crel_data_valid(crel.payload.addr_beat) := Bool(true) } - when(inner_data_done) { collect_inner_data := Bool(false) } + when(crel_data_done) { collect_crel_data := Bool(false) } } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_addr_beat := crel.payload.addr_beat + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_crel_data := crel.payload.hasMultibeatData() + crel_data_valid := Bits(1) state := s_meta_read } } @@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, - Mux(co.messageHasData(xact), s_data_write, s_meta_write), - s_grant) + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_grant, s_idle)) } } is(s_data_write) { - io.data.write.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (local_data_cnt < inner_data_cnt)) - when(local_data_done) { state := s_meta_write } + io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? + } } is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } @@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) + val collect_cacq_data = Reg(init=Bool(false)) + val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) val crel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) val curr_p_id = co.dir.next(pending_probes) val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(co.requiresSelfProbe(xact), + val mask_self = Mux(xact.requiresSelfProbe(), full_sharers | (UInt(1) << xact_src), full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) val mask_incoherent = mask_self & ~io.tile_incoherent - val collect_cacq_data = Reg(init=Bool(false)) - //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = - Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //TODO: uncached does or does not allocate + //val do_allocate = !xact_uncached || xact.allocate() + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := xact.data(0) // default + + def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + amoalu.io.lhs := old_data + amoalu.io.rhs := new_data + val wmask = FillInterleaved(8, xact.write_mask()) + buffer(incoming.addr_beat) := + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), + wmask & new_data | ~wmask & old_data, old_data)) + } - //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || + xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := co.messageHasData(xact) && - (xact.addr === c_acq.payload.addr) && + io.has_acquire_match := xact.hasMultibeatData() && + (xact.addr_block === cacq.payload.addr_block) && collect_cacq_data - io.has_release_match := !co.isVoluntary(c_rel.payload) && - (xact.addr === c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + (xact.addr_block === crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, - c_gnt.header.dst) + val next_coh_on_rel = co.managerMetadataOnRelease( + incoming = crel.payload, + meta = xact_meta.coh, + src = crel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant( + outgoing = cgnt.payload, + meta = xact_meta.coh, + dst = cgnt.header.dst) - val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) + val outer_write = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), + { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) + io.inner.grant.bits.payload := xact.makeGrant( + manager_xact_id = UInt(trackerId), + meta = xact_meta.coh, + addr_beat = cgnt_data_cnt, + data = xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_write_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_gnt io.wb.req.valid := Bool(false) - io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) @@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(cacq_data_cnt) := c_acq.payload.data + xact_data(cacq.payload.addr_beat) := cacq.payload.data + cacq_data_valid(cacq.payload.addr_beat) := Bool(true) } when(cacq_data_done) { collect_cacq_data := Bool(false) } } @@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_cacq_data := co.messageHasData(c_acq.payload) + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_cacq_data := cacq.payload.hasMultibeatData() state := s_meta_read } } @@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) + val full_block = !xact.uncached || + xact.is(Acquire.uncachedReadBlock) || + xact.is(Acquire.uncachedWriteBlock) + read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) + write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) + resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) when(_needs_probes) { pending_probes := mask_incoherent(nCoherentClients-1,0) release_count := co.dir.count(mask_incoherent) @@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_meta.coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - c_rel.payload.data) + mergeData(xact_data, crel.payload) } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasMultibeatData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -847,37 +999,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { //TODO make sure cacq data is actually present before merging - xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), - io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_meta.coh - when(outer_data_resp_done) { - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read) + mergeData(xact_data, mgnt.payload) + when(mgnt_data_done) { + state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) } } } is(s_data_read) { - io.data.read.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) + io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) + } + when(io.data.read.ready) { + read_data_cnt := read_data_cnt + UInt(1) + when(read_data_cnt === read_data_max) { state := s_data_resp } } - when(local_data_read_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) } - when(local_data_resp_done) { - state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + when(resp_data_cnt === resp_data_max) { + state := Mux(xact.hasData(), s_data_write, s_meta_write) } } is(s_data_write) { io.data.write.valid := Bool(true) - when(local_data_write_done) { - state := s_meta_write + when(io.data.write.ready) { + write_data_cnt := write_data_cnt + UInt(1) + when(write_data_cnt === write_data_max) { + state := s_meta_write + } } } is(s_meta_write) { @@ -886,12 +1040,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + when(io.inner.grant.ready) { + cgnt_data_cnt := cgnt_data_cnt + UInt(1) + when(cgnt_data_cnt === cgnt_data_max) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } } - is(s_busy) { + is(s_ack) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 08d7cf20..6367b44d 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,7 +3,6 @@ package uncore import Chisel._ - // Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters @@ -14,9 +13,9 @@ class ClientMetadata extends CoherenceMetadata { } object ClientMetadata { def apply(state: UInt) = { - val m = new ClientMetadata - m.state := state - m + val meta = new ClientMetadata + meta.state := state + meta } } @@ -27,10 +26,10 @@ class ManagerMetadata extends CoherenceMetadata { } object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new ManagerMetadata - m.state := state - m.sharers := sharers - m + val meta = new ManagerMetadata + meta.state := state + meta.sharers := sharers + meta } def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } @@ -85,56 +84,59 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, m: ClientMetadata): Bool - def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: ManagerMetadata): Bool - def isValid (m: ManagerMetadata): Bool + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + val acquireTypesWithData = Nil // Only built-in Acquire types have data for now + val releaseTypesWithData: Vec[UInt] + val grantTypesWithData: Vec[UInt] + + def isValid(meta: ClientMetadata): Bool + def isValid(meta: ManagerMetadata): Bool + + def isHit(cmd: UInt, meta: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), + clientStatesWithWritePermission.contains(meta.state), + clientStatesWithReadPermission.contains(meta.state)) + } + //TODO: Use outer protocol's clientState instead, remove this function: + def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool - def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: ManagerMetadata): Bool + //TODO: Assumes all cache ctrl ops writeback dirty data, and + // doesn't issue transaction when e.g. downgrading Exclusive to Shared: + def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + clientStatesWithDirtyData.contains(meta.state) + def needsWriteback(meta: ClientMetadata): Bool = + needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata - def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata - def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def managerMetadataOnFlush: ManagerMetadata - def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnFlush: ClientMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: ManagerMetadata): UInt - def getProbeTypeOnVoluntaryWriteback: UInt - def getReleaseTypeOnCacheControl(cmd: UInt): UInt - def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: ManagerMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt + def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnFlush: ManagerMetadata - def messageHasData(rel: TileLinkChannel): Bool - def messageUpdatesDataArray(g: Grant): Bool - def isVoluntary(rel: Release): Bool - def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool - def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool - def requiresAckForGrant(g: Grant): Bool - def requiresAckForRelease(r: Release): Bool + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - - def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -142,123 +144,105 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 - def nReleaseTypes = 5 - def nGrantTypes = 2 + def nReleaseTypes = 4 + def nGrantTypes = 1 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) - def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) - def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientValid), Array( - M_INV -> (m.state === clientValid), - M_CLN -> (m.state === clientValid) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (outstanding.a_type != acquireExclusive) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientValid - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, Acquire.uncachedAtomic -> probeInvalidate )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -266,137 +250,123 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 2 + def nReleaseTypes = 6 + def nGrantTypes = 1 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) - def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata) = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientExclusiveClean - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), Bool(true)))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -404,152 +374,133 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 + def nReleaseTypes = 6 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (isRead(cmd) && outstanding.uncached) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> clientExclusiveDirty, + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), - MuxLookup(a.a_type, probeInvalidate, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -557,157 +508,135 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 4 + def nReleaseTypes = 6 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(outstanding.a_type === acquireExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -715,174 +644,170 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 - def nReleaseTypes = 11 - def nGrantTypes = 5 + def nReleaseTypes = 10 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), - M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( - Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> m.state, - probeDowngrade -> MuxLookup(m.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid)) - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireExclusive -> clientExclusiveDirty, + acquireShared -> clientExclusiveClean)), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireExclusive -> clientMigratoryDirty, + acquireShared -> clientMigratoryClean)))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next, r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next - )) - } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + r.is(releaseInvalidateAckMigratory) -> next)) } + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), - releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData - )) + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(meta.state === clientMigratoryDirty, + releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData)) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + probeInvalidate -> Mux(clientExclusiveClean === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(meta.state != clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck)) + Mux(dirty, with_data, without_data) } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? - ))) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index fe647d07..69cdc41f 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,6 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") - val MT_CB = Bits("b111") // cache block val NUM_XA_OPS = 9 val M_SZ = 5 @@ -37,8 +36,9 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); val M_XA_MAXU = Bits("b01111"); - val M_INV = Bits("b10000"); // write back and invalidate line - val M_CLN = Bits("b10001"); // write back line + val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions + val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4db133d6..5e5053be 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -136,7 +136,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) + mem_needs_ack := io.mem.grant.bits.payload.requiresAck() } io.mem.grant.ready := Bool(true) @@ -184,22 +184,26 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: UInt = null - for (i <- 0 until dataBits/short_request_bits) { - val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) + val n = dataBits/short_request_bits + val mem_req_data = (0 until n).map { i => + val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := + io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } - mem_req_data = Cat(packet_ram(idx), mem_req_data) - } + packet_ram(Cat(cnt, ui)) + }.reverse.reduce(_##_) + val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, mem_req_data), - UncachedRead(init_addr)) + UncachedWriteBlock( + addr_block = init_addr, + addr_beat = cnt, + client_xact_id = UInt(0), + data = mem_req_data), + UncachedReadBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id - io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index d6133fc0..e7291222 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -228,7 +228,9 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { mem_cmd_q.io.enq.valid := Bool(false) mem_data_q.io.enq.valid := Bool(false) - val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are + //acceptable Acquire types + val acq_has_data = io.uncached.acquire.bits.payload.hasData() val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) @@ -250,7 +252,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr + addr_out := io.uncached.acquire.bits.payload.addr_block has_data := acq_has_data tl_done_out := tl_wrap_out mif_done_out := Bool(false) @@ -323,8 +325,12 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), - tl_buf_in(tl_cnt_in)) + io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + g_type = Grant.uncachedReadBlock, + client_xact_id = tag_in, + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af10d455..32ccac87 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,20 +16,22 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { - val tlAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits - val tlSubblockAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlBeatAddrBits = log2Up(tlDataBeats) + val tlByteAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = M_SZ val tlUncachedOperandSizeBits = MT_SZ val tlSubblockUnionBits = max(tlWriteMaskBits, - (tlSubblockAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 + (tlByteAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) + val networkPreservesPointToPointOrdering = false //TODO: check physical network type } abstract class TLBundle extends Bundle with TileLinkParameters @@ -45,8 +47,12 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. // These traits are useful for type parameterization. // -trait HasPhysicalAddress extends TLBundle { - val addr = UInt(width = tlAddrBits) +trait HasCacheBlockAddress extends TLBundle { + val addr_block = UInt(width = tlBlockAddrBits) +} + +trait HasTileLinkBeatId extends TLBundle { + val addr_beat = UInt(width = tlBeatAddrBits) } trait HasClientTransactionId extends TLBundle { @@ -57,55 +63,109 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -trait HasTileLinkData extends TLBundle { +abstract trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool } // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { + // Actual bundle fields val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits - val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + + // Utility funcs for accessing uncached/subblock union + val opSizeOff = tlByteAddrBits + 1 + val opCodeOff = tlUncachedOperandSizeBits + opSizeOff + val opMSB = tlAtomicOpcodeBits + opCodeOff def allocate(dummy: Int = 0) = subblock(0) - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) - def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) + def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) + def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + + // Other helper funcs def is(t: UInt) = a_type === t + + def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + Acquire.typesWithMultibeatData.contains(a_type) + + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + + def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = + Probe(co.getProbeType(this, meta), this.addr_block) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Grant( + uncached = this.uncached, + g_type = co.getGrantType(this, meta), + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data + ) + } } object Acquire { - val nUncachedAcquireTypes = 3 - //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite - def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + val nBuiltinAcquireTypes = 5 + //TODO: Use Enum + def uncachedRead = UInt(0) + def uncachedReadBlock = UInt(1) + def uncachedWrite = UInt(2) + def uncachedWriteBlock = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) + def typesWithMultibeatData = Vec(uncachedWriteBlock) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock + def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock - def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { + def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + + // Most generic constructor + def apply( + uncached: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire acq.uncached := uncached acq.a_type := a_type - acq.addr := addr acq.client_xact_id := client_xact_id + acq.addr_block := addr_block + acq.addr_beat := addr_beat acq.data := data acq.subblock := subblock acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { - apply(a_type, addr, client_xact_id, UInt(0)) + // For cached types + def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { + apply( + uncached = Bool(false), + a_type = a_type, + client_xact_id = client_xact_id, + addr_block = addr_block) } + // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a @@ -113,83 +173,189 @@ object Acquire { } } +// Asks for a single TileLink beat of data object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { - val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) - acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedRead, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + subblock = alloc) } - def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) - } - def apply(addr: UInt): Acquire = { - apply(addr, UInt(0)) +} + +// Asks for an entire cache block of data +object UncachedReadBlock { + def apply( + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedReadBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + subblock = alloc.toUInt) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(write_mask, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt = Acquire.fullWriteMask, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWrite, + addr_block = addr_block, + addr_beat = addr_beat, + client_xact_id = client_xact_id, + data = data, + subblock = Cat(write_mask, alloc)) } - def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), Bool(true), data) - } - def apply(addr: UInt, data: UInt): Acquire = { - apply(addr, UInt(0), data) +} + +// For full block of data +object UncachedWriteBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWriteBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(Acquire.fullWriteMask, alloc)) } } object UncachedAtomic { - def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, - subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedAtomic, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) } } class Probe extends ManagerToClientChannel - with HasPhysicalAddress { + with HasCacheBlockAddress { val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t + def makeRelease( + client_xact_id: UInt, + meta: ClientMetadata = co.clientMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(false), + r_type = co.getReleaseType(this, meta), + client_xact_id = client_xact_id, + addr_block = this.addr_block, + addr_beat = addr_beat, + data = data) + } } object Probe { - def apply(p_type: UInt, addr: UInt) = { + val co = new Probe().co + def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type - prb.addr := addr + prb.addr_block := addr_block prb } + + def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { + apply(co.getProbeType(M_FLUSH, meta), addr_block) + } } + class Release extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { val r_type = UInt(width = co.releaseTypeWidth) + val voluntary = Bool() + + // Helper funcs def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { + Grant( + g_type = Grant.voluntaryAck, + uncached = Bool(true), // Grant.voluntaryAck is built-in type + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id + ) + } } object Release { - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { + val co = new Release().co + def apply( + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { val rel = new Release rel.r_type := r_type - rel.addr := addr rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat rel.data := data + rel.voluntary := voluntary rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { - apply(r_type, addr, client_xact_id, UInt(0)) - } - def apply(r_type: UInt, addr: UInt): Release = { - apply(r_type, addr, UInt(0), UInt(0)) + + def makeVoluntaryWriteback( + meta: ClientMetadata, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, meta), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data) } } @@ -198,29 +364,62 @@ class Grant extends ManagerToClientChannel with HasClientTransactionId with HasManagerTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + + // Helper funcs def is(t: UInt) = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(uncached, + Grant.typesWithData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(uncached, + Grant.typesWithMultibeatData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + def makeFinish(dummy: Int = 0): Finish = { + val f = new Finish + f.manager_xact_id := this.manager_xact_id + f + } } object Grant { - val nUncachedGrantTypes = 3 - //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + val nBuiltinGrantTypes = 5 + //TODO Use Enum + def voluntaryAck = UInt(0) + def uncachedRead = UInt(1) + def uncachedReadBlock = UInt(2) + def uncachedWrite = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) + def typesWithMultibeatData= Vec(uncachedReadBlock) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { + def apply( + uncached: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) + + def getGrantTypeForUncached(a: Acquire): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedWriteBlock -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) } } @@ -260,10 +459,17 @@ object TileLinkIOWrapper { } } -// Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { +abstract trait HasArbiterTypes { + val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with + HasClientTransactionId with + HasTileLinkData +} +// Utility functions for constructing TileLinkIO arbiters +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule + with HasArbiterTypes { // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back @@ -271,10 +477,10 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId] + def hookupClientSource[M <: ClientSourcedWithIdAndData] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -336,35 +542,26 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { } // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = +abstract trait AppendsArbiterId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id(log2Up(arbN)-1,0).toUInt + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId { - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - in.client_xact_id - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait PassesId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - UInt(0) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait UsesNewId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Mix-in id generation traits to make concrete arbiter classes diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d2531a9e..c05aafee 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -67,7 +67,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } @@ -87,8 +87,8 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends // Queue to store impending Voluntary Release data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } @@ -117,9 +117,10 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -130,7 +131,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - co.messageHasData(io.outer.acquire.bits.payload) && + io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), @@ -156,32 +157,39 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val has_release_match = Bool(OUTPUT) } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cfin = io.inner.finish.bits + val macq = io.outer.acquire.bits + val mgnt = io.outer.grant.bits } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) val collect_inner_data = Reg(init=Bool(false)) + // TODO: assert that all releases have full blocks of data val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -193,21 +201,19 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact_addr, - UInt(trackerId), - xact_data(outer_data_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := crel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -216,72 +222,107 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) - state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_inner_data := crel.payload.hasMultibeatData() + state := Mux(crel.payload.hasData(), s_outer, + Mux(crel.payload.requiresAck(), s_ack, s_idle)) } } is(s_outer) { io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { state := s_ack } + when(outer_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } } } } class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) val collect_inner_data = Reg(init=Bool(false)) + //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) + val pending_outer_write = xact.hasData() val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && (state != s_idle) && !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - !co.isVoluntary(c_rel.payload) && + io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + !crel.payload.isVoluntary() && (state != s_idle) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + val outer_write_acq = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = crel.payload.addr_beat, + data = crel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -290,16 +331,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe() io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId), - UInt(0)) // Data bypassed in parent + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -307,7 +344,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := cacq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -315,17 +352,18 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) - when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_inner_data := co.messageHasData(c_acq.payload) + val needs_outer_write = cacq.payload.hasData() + val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) + when(io.inner.acquire.valid) { + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_inner_data := cacq.payload.hasMultibeatData() probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, @@ -341,9 +379,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back - io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready + io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready when(io.inner.release.valid) { - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { @@ -364,14 +402,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } - is(s_mem_read) { + is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) - } + when(io.outer.acquire.ready) { state := s_mem_resp } } - is(s_mem_write) { + is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq @@ -379,17 +415,22 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_make_grant) { + is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + is(s_mem_resp) { // Wait to forward grants from outer memory + when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { + when(cgnt_data_done) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index c28778aa..14575aa5 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,7 +34,7 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) @@ -65,12 +65,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(doSer(io.in.bits.payload)) { + when(io.in.bits.payload.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !doSer(io.in.bits.payload) + io.done := !io.in.bits.payload.hasData() } when(active) { io.out.bits := rbits