From fbca7c6bb35914ac0c3f043090d4e8d407306ae2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 29 Mar 2014 10:53:49 -0700 Subject: [PATCH] refactor ioMem and associcated constants. merge Aqcuire and AcquireData --- uncore/src/main/scala/coherence.scala | 17 -- uncore/src/main/scala/consts.scala | 6 - uncore/src/main/scala/htif.scala | 56 +++-- uncore/src/main/scala/llc.scala | 72 +++--- uncore/src/main/scala/memserdes.scala | 141 +++++++++--- uncore/src/main/scala/network.scala | 113 +-------- uncore/src/main/scala/tilelink.scala | 89 ++++--- uncore/src/main/scala/uncore.scala | 320 +++++++++++--------------- 8 files changed, 384 insertions(+), 430 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index e0db85f1..c9d0f4eb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -48,7 +48,6 @@ abstract class CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -79,7 +78,6 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def requiresAckForGrant(g_type: UInt) = Bool(true) def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -282,9 +280,6 @@ class MICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -442,9 +437,6 @@ class MEICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -606,9 +598,6 @@ class MSICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -773,9 +762,6 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -958,9 +944,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index d4f7130c..b0cdcdd6 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -70,9 +70,3 @@ trait TileLinkSizeConstants { trait MemoryInterfaceConstants extends CacheConstants with AddressConstants -{ - val MEM_TAG_BITS = 5 - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS -} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 608ac051..c4d0ffb0 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES + val long_request_bits = short_request_bits + conf.dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -124,26 +124,23 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati } io.mem.grant.ready := Bool(true) - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) val state = Reg(init=state_rx) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { - state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, + Mux(rx_cmd === cmd_writemem, state_mem_wreq, Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, - state_tx)) + state_tx))) } - val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val x_init = Module(new Queue(new Acquire, 1)) - when (state === state_mem_req && x_init.io.enq.ready) { - state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + val acq_q = Module(new Queue(new Acquire, 1)) + when (state === state_mem_wreq && acq_q.io.enq.ready) { + state := state_mem_wresp } - when (state === state_mem_wdata && io.mem.acquire.data.ready) { - when (mem_cnt.andR) { - state := state_mem_wresp - } - mem_cnt := mem_cnt + UInt(1) + when (state === state_mem_rreq && acq_q.io.enq.ready) { + state := state_mem_rresp } when (state === state_mem_wresp) { when (mem_acked) { @@ -151,12 +148,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(false) } } - when (state === state_mem_rdata) { + when (state === state_mem_rresp) { when (io.mem.grant.valid) { - when (mem_cnt.andR) { - state := state_mem_finish - } - mem_cnt := mem_cnt + UInt(1) + state := state_mem_finish } mem_acked := Bool(false) } @@ -170,31 +164,33 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati rx_count := UInt(0) tx_count := UInt(0) } - state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } var mem_req_data: Bits = null - for (i <- 0 until MEM_DATA_BITS/short_request_bits) { - val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) - when (state === state_mem_rdata && io.mem.grant.valid) { + for (i <- 0 until conf.dataBits/short_request_bits) { + val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - x_init.io.enq.valid := state === state_mem_req + acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) - x_init.io.enq.bits := Mux(cmd === cmd_writemem, + acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) - io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id - io.mem.acquire.data.valid := state === state_mem_wdata - io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.acquire.valid := acq_q.io.deq.valid + acq_q.io.deq.ready := io.mem.acquire.ready + io.mem.acquire.bits.payload := acq_q.io.deq.bits + io.mem.acquire.bits.payload.data := mem_req_data + io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid io.mem.grant_ack.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) - io.mem.release.meta.valid := Bool(false) - io.mem.release.data.valid := Bool(false) + io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) @@ -244,7 +240,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 6627527a..1bfa4da7 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,42 +72,42 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int) extends MemReqCmd +class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] } -class LLCTagReq(ways: Int) extends HasMemAddr +class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) - val mem = new ioMemPipe + val mem = new MemPipeIO val mem_resp_set = UInt(OUTPUT, log2Up(sets)) val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UInt(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = conf.addrBits - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -139,7 +139,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -183,12 +183,12 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new ioMemPipe + val mem = new MemPipeIO } val valid = Reg(init=Bool(false)) @@ -196,7 +196,7 @@ class LLCWriteback(requestors: Int) extends Module val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -216,7 +216,7 @@ class LLCWriteback(requestors: Int) extends Module when (io.mem.req_data.valid && io.mem.req_data.ready) { count := count + UInt(1) - when (count === UInt(REFILL_CYCLES-1)) { + when (count === UInt(refill_cycles-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int) extends Module io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback = Decoupled(UInt(width = conf.addrBits)) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -254,12 +254,12 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val qReady = q.io.count <= UInt(q.entries-latency-1) val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) + val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) - when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } + when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } @@ -300,11 +300,11 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Module // UNTESTED +class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = Vec.fill(n){new ioMem().flip} - val mem = new ioMem + val cpu = Vec.fill(n){new MemIO().flip} + val mem = new MemIO } val lock = Reg(init=Bool(false)) @@ -339,26 +339,26 @@ class MemReqArb(n: Int) extends Module // UNTESTED } io.mem.resp.ready := io.cpu(respWho).resp.ready - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } - val tagWidth = MEM_ADDR_BITS - log2Up(sets) + val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, dataLeaf)) - val writeback = Module(new LLCWriteback(2)) + val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) + val writeback = Module(new LLCWriteback(2, refill_cycles)) val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) @@ -497,11 +497,11 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } val numEntries = numRequests * refillCycles diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 42611bc6..21a987fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,49 +2,56 @@ package uncore import Chisel._ import scala.math._ -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) +case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) + +abstract trait MemoryIFSubBundle extends Bundle { + implicit val conf: MemoryIFConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } -trait HasMemAddr extends Bundle { - val addr = UInt(width = MEM_ADDR_BITS) +trait HasMemData extends MemoryIFSubBundle { + val data = Bits(width = conf.dataBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = MEM_TAG_BITS) +trait HasMemAddr extends MemoryIFSubBundle { + val addr = UInt(width = conf.addrBits) } -class MemReqCmd extends HasMemAddr with HasMemTag { +trait HasMemTag extends MemoryIFSubBundle { + val tag = UInt(width = conf.tagBits) +} + +class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag +class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag -class MemData extends HasMemData +class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData -class ioMem extends Bundle { +class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class ioMemPipe extends Bundle { +class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class ioMemSerialized(w: Int) extends Bundle +class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val wide = new MemIO().flip + val narrow = new MemSerializedIO(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -56,7 +63,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -92,12 +99,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -115,12 +122,12 @@ class MemSerdes(w: Int) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int) extends Bundle { - val narrow = new ioMemSerialized(w).flip - val wide = new ioMem +class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { + val narrow = new MemSerializedIO(w).flip + val wide = new MemIO } -class MemDesser(w: Int) extends Module // test rig side +class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -129,7 +136,7 @@ class MemDesser(w: Int) extends Module // test rig side require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -157,13 +164,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -177,10 +184,92 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) + val dataq = Module(new Queue(new MemResp, conf.dataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemIO + } + + require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) + //require(tlconf.clientXactIdBits <= mifconf.tagBits) + + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + val cnt_max = mifconf.dataBeats + + val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val buf_out = Reg(Bits()) + val tag_out = Reg(Bits()) + val addr_out = Reg(Bits()) + val has_data = Reg(init=Bool(false)) + + val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) + val active_in = Reg(init=Bool(false)) + val buf_in = Reg(Bits()) + val tag_in = Reg(UInt(width = mifconf.tagBits)) + + // Decompose outgoing TL Acquires into MemIO cmd and data + when(!active_out && io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + cnt_out := UInt(0) + buf_out := io.uncached.acquire.bits.payload.data + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + } + when(active_out) { + when(mem_cmd_q.io.enq.fire()) { + cmd_sent_out := Bool(true) + } + when(mem_data_q.io.enq.fire()) { + cnt_out := cnt_out + UInt(1) + buf_out := buf_out >> UInt(mifconf.dataBits) + } + when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + active_out := Bool(false) + } + } + + io.uncached.acquire.ready := !active_out + mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) + mem_data_q.io.enq.bits.data := buf_out + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + + // Aggregate incoming MemIO responses into TL Grants + io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) + io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) + io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + when(!active_in && io.mem.resp.valid) { + active_in := Bool(true) + cnt_in := UInt(1) + buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + tag_in := io.mem.resp.bits.tag + } + when(active_in) { + when(io.uncached.grant.fire()) { + active_in := Bool(false) + } + when(io.mem.resp.fire()) { + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + cnt_in := cnt_in + UInt(1) + } + } +} + diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 613a3b65..6889bd7f 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,102 +1,5 @@ package uncore import Chisel._ -import scala.collection.mutable.Stack - -class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { - val meta = Decoupled(mType) - val data = Decoupled(dType) - override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { - val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip - val out = new PairedDataIO(mType, dType) - val meta_chosen = UInt(OUTPUT, log2Up(n)) - val data_chosen = UInt(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { - require(isPow2(count)) - val io = new PairedArbiterIO(mType, dType, n) - val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(init=UInt(0, width = log2Up(count))) - val cnt_next = cnt + UInt(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UInt(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(init=Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} -} - -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new PairedCrossbarIO(mType, dType) - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) @@ -147,7 +50,13 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UInt(width = conf.idBits) } -object FIFOedLogicalNetworkIOWrapper { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } +} + +object DecoupledLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid @@ -159,7 +68,7 @@ object FIFOedLogicalNetworkIOWrapper { } } -object FIFOedLogicalNetworkIOUnwrapper { +object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid @@ -168,9 +77,3 @@ object FIFOedLogicalNetworkIOUnwrapper { out } } - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = dType.clone - override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } -} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index cd50213f..6f877856 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -35,18 +35,24 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.data := Bits(0) acq.write_mask := Bits(0) acq.subword_addr := Bits(0) acq.atomic_opcode := Bits(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) + acq.data := data + acq + } + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { - val acq = apply(a_type, addr, client_xact_id) + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq @@ -57,15 +63,17 @@ object Acquire acq } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData - object Probe { def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -76,16 +84,20 @@ object Probe prb } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasPhysicalAddress + with HasMasterTransactionId { val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr + rel.data := data rel } def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -94,16 +106,45 @@ object Release rel.addr := addr rel.client_xact_id := client_xact_id rel.master_xact_id := master_xact_id + rel.data := UInt(0) + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + val rel = apply(r_type, addr, client_xact_id, master_xact_id) + rel.data := data rel } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasMasterTransactionId + with HasTileLinkData { val r_type = UInt(width = conf.co.releaseTypeWidth) } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Grant +{ + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val gnt = new Grant + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := UInt(0) + gnt + } + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + val gnt = apply(g_type, client_xact_id, master_xact_id) + gnt.data := data + gnt + } +} -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasTileLinkData + with HasClientTransactionId + with HasMasterTransactionId { val g_type = UInt(width = conf.co.grantTypeWidth) } @@ -112,7 +153,7 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -120,7 +161,7 @@ class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val release = new DecoupledIO(new LogicalNetworkIO(new Release)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -129,32 +170,28 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId - type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def masterSourcedClientXactId(in: MasterSourcedWithId): Bits def arbIdx(in: MasterSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] - (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], - out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hookupClientSource[M <: ClientSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) - val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, - arbN, REFILL_CYCLES, hasData _)) + val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload := req.meta.bits.payload - arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready }} } def hookupMasterSource[M <: MasterSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) for (i <- 0 until arbN) { ins(i).valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 67dfde85..3e162583 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -31,16 +31,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.meta.ready - t.acquire.meta.bits := acquire.meta.bits - t.acquire.meta.valid := alloc_arb.io.in(i).ready - - t.acquire.data.bits := acquire.data.bits - t.acquire.data.valid := acquire.data.valid + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires - acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -49,21 +45,18 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Handle releases, which might be voluntary and might have data val release = io.client.release - val voluntary = co.isVoluntary(release.meta.bits.payload) + val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases - t.release.data.bits := release.data.bits - t.release.data.valid := release.data.valid + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases - release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) @@ -83,7 +76,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -92,63 +85,56 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(init=Bool(false)) - val mem_cmd_sent = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := cmd_to_write - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) // DNC + io.client.release.ready := Bool(false) io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.payload.data := UInt(0) - io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.meta.ready := Bool(true) - when( io.client.release.meta.valid ) { - xact := io.client.release.meta.bits.payload - init_client_id_ := io.client.release.meta.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UInt(0) - mem_cmd_sent := Bool(false) - state := s_mem + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - mem_cmd_sent, - init_client_id_) - } . otherwise { state := s_ack } + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } } is(s_ack) { io.client.grant.valid := Bool(true) @@ -158,172 +144,138 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(init=Bool(false)) - val acquire_data_needs_write = Reg(init=Bool(false)) - val release_data_needs_write = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = Reg(init=Bool(false)) - val r_w_mem_cmd_sent = Reg(init=Bool(false)) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) - io.client.probe.bits.payload.master_xact_id := UInt(trackerId) - io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UInt(0) - io.client.probe.bits.header.src := UInt(bankId) - io.client.grant.bits.payload.data := io.master.grant.bits.payload.data - io.client.grant.bits.payload.g_type := grant_type - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read io.master.grant.ready := io.client.grant.ready + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + switch (state) { is(s_idle) { - io.client.acquire.meta.ready := Bool(true) - when( io.client.acquire.meta.valid ) { - xact := io.client.acquire.meta.bits.payload - init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - mem_cnt := UInt(0) - r_w_mem_cmd_sent := Bool(false) - a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) } } is(s_probe) { - when(probe_flags.orR) { - io.client.probe.valid := Bool(true) - io.client.probe.bits.header.dst := curr_p_id - } + // Generate probes + io.client.probe.valid := probe_flags.orR when(io.client.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } - io.client.release.meta.ready := Bool(true) - when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := s_mem - } - when( co.messageHasData(io.client.release.meta.bits.payload)) { - release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.meta.bits.header.src + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } - is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) - } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.acquire.data, - cmd_to_write, - acquire_data_needs_write, - a_w_mem_cmd_sent, - init_client_id_) - } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) - } . otherwise { - state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_ack) { + is(s_make_grant) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } } } - -abstract trait OuterRequestGenerator { - val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UInt(1) - - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { - val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd - master_acq.data.bits.payload := client_data.bits.payload - when(master_acq.meta.fire()) { - cmd_sent := Bool(true) - } - when (do_write) { - master_acq.meta.valid := !cmd_sent - when (master_acq.meta.ready || cmd_sent) { - master_acq.data.valid := client_data.valid - when(master_acq.data.ready) { - client_data.ready:= Bool(true) - mem_cnt := mem_cnt_next - when(mem_cnt === UInt(REFILL_CYCLES-1)) { - trigger := Bool(false) - } - } - } - } - } - - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { - master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd - when(master_acq.meta.ready) { - trigger := Bool(false) - } - } -}