diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b1bf26b..ec4c15df 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,7 +16,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) - val tlWriteMaskBits = tlDataBits/8 + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) val tlUncachedOperandSizeBits = MT_SZ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6ea8cd25..858d603c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -5,14 +5,17 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) - val nReleaseTransactors = params(NReleaseTransactors) + val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) + val sdqDepth = params(L2StoreDataQueueDepth) + val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { + // Queue to store impending UncachedWrite data + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} + when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId))) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -67,12 +79,19 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } + when(voluntary && release.fire()) { + vwbdq(release_idx) := release.bits.payload.data + } + // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) + { case TLId => outerId; case TLDataBits => sdqIdBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) + val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) + val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) + val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) + val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq + io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), + Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } } @@ -112,10 +144,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -140,11 +171,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute switch (state) { is(s_idle) { - io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + io.inner.release.ready := Bool(true) + xact := c_rel.payload + init_client_id := c_rel.header.src + state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) } } is(s_mem) { @@ -173,7 +204,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) @@ -208,7 +239,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + UInt(0)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false)