diff --git a/uncore/src/main/scala/builder.scala b/uncore/src/main/scala/Builder.scala similarity index 98% rename from uncore/src/main/scala/builder.scala rename to uncore/src/main/scala/Builder.scala index 74cb7fa1..ba2c296a 100644 --- a/uncore/src/main/scala/builder.scala +++ b/uncore/src/main/scala/Builder.scala @@ -3,6 +3,9 @@ package uncore import Chisel._ import cde.{Config, Parameters, ParameterDump, Knob, Dump} import junctions.PAddrBits +import uncore.tilelink._ +import uncore.agents._ +import uncore.coherence._ object UncoreBuilder extends App with FileSystemUtilities { val topModuleName = args(0) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/Consts.scala similarity index 99% rename from uncore/src/main/scala/consts.scala rename to uncore/src/main/scala/Consts.scala index 14904463..a4a4e93b 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/Consts.scala @@ -47,3 +47,4 @@ trait MemoryOpConstants { def isWrite(cmd: UInt) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: UInt) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } + diff --git a/uncore/src/main/scala/Package.scala b/uncore/src/main/scala/Package.scala new file mode 100644 index 00000000..c9a35dbb --- /dev/null +++ b/uncore/src/main/scala/Package.scala @@ -0,0 +1,4 @@ +// See LICENSE for license details. +package uncore + +package object constants extends uncore.constants.MemoryOpConstants diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/agents/Agents.scala similarity index 95% rename from uncore/src/main/scala/uncore.scala rename to uncore/src/main/scala/agents/Agents.scala index be20394d..15bf0f7b 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/agents/Agents.scala @@ -1,19 +1,18 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ import cde.{Parameters, Field} import junctions._ +import uncore.tilelink._ +import uncore.converters._ +import uncore.coherence._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -/** Identifies the TLId of the inner network in a hierarchical cache controller */ -case object InnerTLId extends Field[String] -/** Identifies the TLId of the outer network in a hierarchical cache controller */ -case object OuterTLId extends Field[String] - trait HasCoherenceAgentParameters { implicit val p: Parameters val nReleaseTransactors = 1 @@ -49,7 +48,7 @@ trait HasCoherenceAgentWiringHelpers { out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { def lock(o: T) = o.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, lock _)) + val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, Some(lock _))) out <> arb.io.out arb.io.in <> ins } diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala similarity index 97% rename from uncore/src/main/scala/broadcast.scala rename to uncore/src/main/scala/agents/Broadcast.scala index 3665ceaa..cedda9b9 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { @@ -35,7 +39,7 @@ class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) // Handle releases, which might be voluntary and might have data doInputRoutingWithAllocation( diff --git a/uncore/src/main/scala/bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala similarity index 96% rename from uncore/src/main/scala/bufferless.scala rename to uncore/src/main/scala/agents/Bufferless.scala index eca709e4..959367f8 100644 --- a/uncore/src/main/scala/bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { @@ -36,7 +40,7 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) io.outer.acquire.bits.data := io.inner.acquire.bits.data io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/agents/Cache.scala similarity index 99% rename from uncore/src/main/scala/cache.scala rename to uncore/src/main/scala/agents/Cache.scala index 5464dd1c..6f7e1cae 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -1,9 +1,14 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ import scala.reflect.ClassTag import junctions._ +import uncore.util.AMOALU +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ import cde.{Parameters, Field} case object CacheName extends Field[String] @@ -12,11 +17,9 @@ case object NWays extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object L2Replacer extends Field[() => SeqReplacementPolicy] -case object AmoAluOperandBits extends Field[Int] case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] -case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] case object CacheIdBits extends Field[Int] case object CacheId extends Field[Int] @@ -477,7 +480,7 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) assert(PopCount(trackerList.map(_.io.alloc.iacq.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Acquire") diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/agents/Ecc.scala similarity index 99% rename from uncore/src/main/scala/ecc.scala rename to uncore/src/main/scala/agents/Ecc.scala index 4a582fc1..6e5fdba6 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/agents/Ecc.scala @@ -1,6 +1,6 @@ // See LICENSE for license details. -package uncore +package uncore.agents import Chisel._ diff --git a/uncore/src/main/scala/agents/Mmio.scala b/uncore/src/main/scala/agents/Mmio.scala new file mode 100644 index 00000000..a3b2ab03 --- /dev/null +++ b/uncore/src/main/scala/agents/Mmio.scala @@ -0,0 +1,73 @@ +package uncore.agents + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +class MMIOTileLinkManagerData(implicit p: Parameters) + extends TLBundle()(p) + with HasClientId + with HasClientTransactionId + +class MMIOTileLinkManager(implicit p: Parameters) + extends CoherenceAgentModule()(p) { + val io = new ManagerTLIO + + // MMIO requests should never need probe or release + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + + val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData() + val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0) + val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1) + + // Acquire and Grant are basically passthru, + // except client_id and client_xact_id need to be converted. + // Associate the inner client_id and client_xact_id + // with the outer client_xact_id. + val xact_pending = Reg(init = UInt(0, maxManagerXacts)) + val xact_id_sel = PriorityEncoder(~xact_pending) + val xact_id_reg = RegEnable(xact_id_sel, multibeat_start) + val xact_multibeat = Reg(init = Bool(false)) + val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel) + val xact_free = !xact_pending.andR + val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData)) + + io.inner.acquire.ready := io.outer.acquire.ready && xact_free + io.outer.acquire.valid := io.inner.acquire.valid && xact_free + io.outer.acquire.bits := io.inner.acquire.bits + io.outer.acquire.bits.client_xact_id := outer_xact_id + + def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool = + !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1) + + def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt = + Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0)) + + def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt = + ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(), + UIntToOH(in.bits.manager_xact_id), UInt(0)) + + def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt = + ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0)) + + xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) & + clearPendingBitOnFin(io.inner.finish) & + clearPendingBitOnGnt(io.inner.grant) + + when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) { + xact_buffer(outer_xact_id) := io.iacq() + } + + when (multibeat_start) { xact_multibeat := Bool(true) } + when (multibeat_end) { xact_multibeat := Bool(false) } + + val gnt_xact = xact_buffer(io.ognt().client_xact_id) + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + io.inner.grant.bits := io.outer.grant.bits + io.inner.grant.bits.client_id := gnt_xact.client_id + io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id + io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id + io.inner.finish.ready := Bool(true) +} diff --git a/uncore/src/main/scala/sdq.scala b/uncore/src/main/scala/agents/StoreDataQueue.scala similarity index 98% rename from uncore/src/main/scala/sdq.scala rename to uncore/src/main/scala/agents/StoreDataQueue.scala index 6b2cc092..e2079772 100644 --- a/uncore/src/main/scala/sdq.scala +++ b/uncore/src/main/scala/agents/StoreDataQueue.scala @@ -1,7 +1,8 @@ // See LICENSE for license details. -package uncore +package uncore.agents import Chisel._ +import uncore.tilelink._ import cde.{Parameters, Field} case object L2StoreDataQueueDepth extends Field[Int] diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/agents/Trackers.scala similarity index 99% rename from uncore/src/main/scala/trackers.scala rename to uncore/src/main/scala/agents/Trackers.scala index 25323d3c..13bcb70d 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.util._ +import cde.Parameters import scala.math.max class TrackerAllocation extends Bundle { diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/coherence/Directory.scala similarity index 98% rename from uncore/src/main/scala/directory.scala rename to uncore/src/main/scala/coherence/Directory.scala index db555ad3..86e4fde5 100644 --- a/uncore/src/main/scala/directory.scala +++ b/uncore/src/main/scala/coherence/Directory.scala @@ -1,6 +1,6 @@ // See LICENSE for license details. -package uncore +package uncore.coherence import Chisel._ // This class encapsulates transformations on different directory information diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/coherence/Metadata.scala similarity index 97% rename from uncore/src/main/scala/metadata.scala rename to uncore/src/main/scala/coherence/Metadata.scala index 38c51428..c0d7a6bf 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/coherence/Metadata.scala @@ -1,9 +1,17 @@ // See LICENSE for license details. -package uncore +package uncore.coherence + import Chisel._ +import uncore.tilelink._ +import uncore.constants._ import cde.{Parameters, Field} +/** Identifies the TLId of the inner network in a hierarchical cache controller */ +case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ +case object OuterTLId extends Field[String] + /** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata(implicit p: Parameters) extends TLBundle()(p) { val co = tlCoh diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence/Policies.scala similarity index 99% rename from uncore/src/main/scala/coherence.scala rename to uncore/src/main/scala/coherence/Policies.scala index 3bf224e9..a28e4177 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence/Policies.scala @@ -1,7 +1,10 @@ // See LICENSE for license details. -package uncore +package uncore.coherence + import Chisel._ +import uncore.tilelink._ +import uncore.constants._ /** The entire CoherencePolicy API consists of the following three traits: * HasCustomTileLinkMessageTypes, used to define custom messages diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala deleted file mode 100644 index 93b1372c..00000000 --- a/uncore/src/main/scala/converters.scala +++ /dev/null @@ -1,1226 +0,0 @@ -package uncore - -import Chisel._ -import junctions._ -import cde.Parameters - -/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ -object TileLinkIOWrapper { - def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { - val conv = Module(new ClientTileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl - def apply(tl: TileLinkIO): TileLinkIO = tl -} - -class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse - * to [[uncore.Grant]] that it receives from a manager and forwards to a client - */ -class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) - with HasDataBeatCounters { - val io = new Bundle { - val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip - val refill = Decoupled(new Grant) - val finish = Decoupled(new LogicalNetworkIO(new Finish)) - val ready = Bool(OUTPUT) - } - - val g = io.grant.bits.payload - - if(tlNetworkPreservesPointToPointOrdering) { - io.finish.valid := Bool(false) - io.refill.valid := io.grant.valid - io.refill.bits := g - io.grant.ready := io.refill.ready - io.ready := Bool(true) - } else { - // We only want to send Finishes after we have collected all beats of - // a multibeat Grant. But Grants from multiple managers or transactions may - // get interleaved, so we could need a counter for each. - val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounterWithHeader(io.grant) - } else { - val entries = 1 << tlClientXactIdBits - def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id - assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) - } - val q = Module(new FinishQueue(outstanding)) - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits := g.makeFinish() - q.io.enq.bits.manager_id := io.grant.bits.header.src - - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.manager_id - io.finish.bits.payload := q.io.deq.bits - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid - io.refill.bits := g - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready - io.ready := q.io.enq.ready - } -} - -class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries) - -/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] - * - * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Probe Probes]]. - * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response, - * setting up the headers for each. - * - * @param clientId network port id of this agent - * @param addrConvert how a physical address maps to a destination manager port id - */ -class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val client = new ClientTileLinkIO().flip - val network = new TileLinkIO - } - - val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) - val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId) - val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) - val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant) - - io.network.acquire <> acq_with_header - io.network.release <> rel_with_header - io.network.finish <> fin_with_header - io.client.probe <> prb_without_header - io.client.grant.bits.manager_id := io.network.grant.bits.header.src - io.client.grant <> gnt_without_header -} - -/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]] - * - * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, - * calculating header.dst and filling in header.src. - * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. - * - * @param clientId network port id of this agent - * @param addrConvert how a physical address maps to a destination manager port id - */ -class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val client = new ClientUncachedTileLinkIO().flip - val network = new TileLinkIO - } - - val finisher = Module(new FinishUnit(clientId)) - finisher.io.grant <> io.network.grant - io.network.finish <> finisher.io.finish - - val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val gnt_without_header = finisher.io.refill - - io.network.acquire.bits := acq_with_header.bits - io.network.acquire.valid := acq_with_header.valid && finisher.io.ready - acq_with_header.ready := io.network.acquire.ready && finisher.io.ready - io.client.grant <> gnt_without_header - io.network.probe.ready := Bool(false) - io.network.release.valid := Bool(false) -} - -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasManagerId]( - in: DecoupledIO[T], - clientId: Int) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := in.bits.manager_id - out.valid := in.valid - in.ready := out.ready - out - } - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } -} - -/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip - * - * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], - * but supplies client_id instead. - * - * @param managerId the network port id of this agent - * @param idConvert how a sharer id maps to a destination client port id - */ -class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val manager = new ManagerTileLinkIO().flip - val network = new TileLinkIO().flip - } - io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) - io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) - io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) - io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src - io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) - io.manager.release.bits.client_id := io.network.release.bits.header.src - io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) -} - -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} - -class BeatCounterStatus extends Bundle { - val idx = UInt() - val done = Bool() -} - -class TwoWayBeatCounterStatus extends Bundle { - val pending = Bool() - val up = new BeatCounterStatus() - val down = new BeatCounterStatus() -} - -/** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. - * - * Won't count message types that don't have data. - * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. - */ -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - type HasId = TileLinkChannel with HasClientId - - /** Returns the current count on this channel and when a message is done - * @param inc increment the counter (usually .valid or .fire()) - * @param data the actual channel data - * @param beat count to return for single-beat messages - */ - def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ - def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( - out: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = - connectDataBeatCounter(out.fire(), out.bits, beat) - - /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on - * incoming channels in case of network reordering. - */ - def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = - connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 - - /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ - def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - - /** If the network might interleave beats from different messages, we need a Vec of counters, - * one for every outstanding message id that might be interleaved. - * - * @param getId mapping from Message to counter id - */ - def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - /** Provides counters on two channels, as well a meta-counter that tracks how many - * messages have been sent over the up channel but not yet responded to over the down channel - * - * @param status bundle of status of the counters - * @param up outgoing channel - * @param down incoming channel - * @param max max number of outstanding ups with no down - * @param beat overrides cnts on single-beat messages - * @param track whether up's message should be tracked - * @return a tuple containing whether their are outstanding messages, up's count, - * up's done, down's count, down's done - */ - def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel]( - status: TwoWayBeatCounterStatus, - up: DecoupledIO[T], - down: DecoupledIO[S], - max: Int = 1, - beat: UInt = UInt(0), - trackUp: T => Bool = (t: T) => Bool(true), - trackDown: S => Bool = (s: S) => Bool(true)) { - val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat) - val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat) - val cnt = TwoWayCounter(up_done, dn_done, max) - status.pending := cnt > UInt(0) - status.up.idx := up_idx - status.up.done := up_done - status.down.idx := dn_idx - status.down.done := dn_done - } -} - -class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientTileLinkIO().flip - val out = new ClientUncachedTileLinkIO - } - - val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, - Some((acq: Acquire) => acq.hasMultibeatData()))) - - val acqRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val relRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val iacq = io.in.acquire.bits - val irel = io.in.release.bits - val ognt = io.out.grant.bits - - val acq_roq_enq = iacq.first() - val rel_roq_enq = irel.first() - - val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready - val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready - - val acq_helper = DecoupledHelper( - io.in.acquire.valid, - acq_roq_ready, - acqArb.io.in(0).ready) - - val rel_helper = DecoupledHelper( - io.in.release.valid, - rel_roq_ready, - acqArb.io.in(1).ready) - - acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) - acqRoq.io.enq.bits.data := iacq.isBuiltInType() - acqRoq.io.enq.bits.tag := iacq.client_xact_id - - acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) - acqArb.io.in(0).bits := Acquire( - is_builtin_type = Bool(true), - a_type = Mux(iacq.isBuiltInType(), - iacq.a_type, Acquire.getBlockType), - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = iacq.addr_beat, - data = iacq.data, - union = Mux(iacq.isBuiltInType(), - iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) - io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) - - relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) - relRoq.io.enq.bits.data := irel.isVoluntary() - relRoq.io.enq.bits.tag := irel.client_xact_id - - acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) - acqArb.io.in(1).bits := PutBlock( - client_xact_id = irel.client_xact_id, - addr_block = irel.addr_block, - addr_beat = irel.addr_beat, - data = irel.data, - wmask = Acquire.fullWriteMask) - io.in.release.ready := rel_helper.fire(io.in.release.valid) - - io.out.acquire <> acqArb.io.out - - val grant_deq_roq = io.out.grant.fire() && ognt.last() - - acqRoq.io.deq.valid := acqRoq.io.deq.matches && grant_deq_roq - acqRoq.io.deq.tag := ognt.client_xact_id - - relRoq.io.deq.valid := !acqRoq.io.deq.matches && grant_deq_roq - relRoq.io.deq.tag := ognt.client_xact_id - - assert(!grant_deq_roq || acqRoq.io.deq.matches || relRoq.io.deq.matches, - "TileLink Unwrapper: client_xact_id mismatch") - - val gnt_builtin = acqRoq.io.deq.data - val gnt_voluntary = relRoq.io.deq.data - - val acq_grant = Grant( - is_builtin_type = gnt_builtin, - g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.") - val rel_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - io.in.grant.valid := io.out.grant.valid - io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) - io.out.grant.ready := io.in.grant.ready - - io.in.probe.valid := Bool(false) -} - -class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module - with HasTileLinkParameters with HasNastiParameters { - val io = new Bundle { - val req = new Bundle { - val valid = Bool(INPUT) - val ready = Bool(OUTPUT) - val tl_id = UInt(INPUT, tlClientXactIdBits) - val nasti_id = UInt(OUTPUT, nastiXIdBits) - } - val resp = new Bundle { - val valid = Bool(INPUT) - val matches = Bool(OUTPUT) - val nasti_id = UInt(INPUT, nastiXIdBits) - val tl_id = UInt(OUTPUT, tlClientXactIdBits) - } - } - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort - - if (tlClientXactIdBits <= nastiXIdBits) { - io.req.ready := Bool(true) - io.req.nasti_id := io.req.tl_id - io.resp.matches := Bool(true) - io.resp.tl_id := io.resp.nasti_id - } else if (nastiXIdBits <= 2) { - val nQueues = 1 << nastiXIdBits - val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) - io.req.ready := Bool(false) - io.resp.matches := Bool(false) - io.resp.tl_id := UInt(0) - io.req.nasti_id := req_nasti_id - for (i <- 0 until nQueues) { - val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) - queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) - queue.io.enq.bits := io.req.tl_id - when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } - - queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) - when (io.resp.nasti_id === UInt(i)) { - io.resp.matches := queue.io.deq.valid - io.resp.tl_id := queue.io.deq.bits - } - } - } else { - val maxNastiId = 1 << nastiXIdBits - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) - val roq = Module(new ReorderQueue( - UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) - roq.io.enq.valid := io.req.valid - roq.io.enq.bits.data := io.req.tl_id - roq.io.enq.bits.tag := req_nasti_id - io.req.ready := roq.io.enq.ready - io.req.nasti_id := req_nasti_id - roq.io.deq.valid := io.resp.valid - roq.io.deq.tag := io.resp.nasti_id - io.resp.tl_id := roq.io.deq.data - io.resp.matches := roq.io.deq.matches - } -} - -class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { - val addr_beat = UInt(width = tlBeatAddrBits) - val subblock = Bool() -} - -class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) - with HasNastiParameters { - val io = new Bundle { - val tl = new ClientUncachedTileLinkIO().flip - val nasti = new NastiIO - } - - private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( - MT_B -> UInt(0), - MT_BU -> UInt(0), - MT_H -> UInt(1), - MT_HU -> UInt(1), - MT_W -> UInt(2), - MT_WU -> UInt(2), - MT_D -> UInt(3), - MT_Q -> UInt(log2Up(tlDataBytes)))) - - val dataBits = tlDataBits*tlDataBeats - require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction - require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - - val has_data = io.tl.acquire.bits.hasData() - - val is_subblock = io.tl.acquire.bits.isSubBlockType() - val is_multibeat = io.tl.acquire.bits.hasMultibeatData() - val (tl_cnt_out, tl_wrap_out) = Counter( - io.tl.acquire.fire() && is_multibeat, tlDataBeats) - - val get_valid = io.tl.acquire.valid && !has_data - val put_valid = io.tl.acquire.valid && has_data - - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort - - // Reorder queue saves extra information needed to send correct - // grant back to TL client - val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) - - val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) - val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) - - val get_id_ready = get_id_mapper.io.req.ready - val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) - val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask - - // For Get/GetBlock, make sure Reorder queue can accept new entry - val get_helper = DecoupledHelper( - get_valid, - roq.io.enq.ready, - io.nasti.ar.ready, - get_id_ready) - - val w_inflight = Reg(init = Bool(false)) - - // For Put/PutBlock, make sure aw and w channel are both ready before - // we send the first beat - val aw_ready = w_inflight || io.nasti.aw.ready - val put_helper = DecoupledHelper( - put_valid, - aw_ready, - io.nasti.w.ready, - put_id_ready) - - val (nasti_cnt_out, nasti_wrap_out) = Counter( - io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - - roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) - roq.io.enq.bits.tag := io.nasti.ar.bits.id - roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat - roq.io.enq.bits.data.subblock := is_subblock - roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) - roq.io.deq.tag := io.nasti.r.bits.id - - get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) - get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id - get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last - get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id - - put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) - put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id - put_id_mapper.io.resp.valid := io.nasti.b.fire() - put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id - - // Decompose outgoing TL Acquires into Nasti address and data channels - io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) - io.nasti.ar.bits := NastiReadAddressChannel( - id = get_id_mapper.io.req.nasti_id, - addr = io.tl.acquire.bits.full_addr(), - size = Mux(is_subblock, - opSizeToXSize(io.tl.acquire.bits.op_size()), - UInt(log2Ceil(tlDataBytes))), - len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) - - io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) - io.nasti.aw.bits := NastiWriteAddressChannel( - id = put_id_mapper.io.req.nasti_id, - addr = io.tl.acquire.bits.full_addr(), - size = UInt(log2Ceil(tlDataBytes)), - len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) - - io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) - io.nasti.w.bits := NastiWriteDataChannel( - id = put_id_mapper.io.req.nasti_id, - data = io.tl.acquire.bits.data, - strb = io.tl.acquire.bits.wmask(), - last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) - - io.tl.acquire.ready := Mux(has_data, - put_helper.fire(put_valid), - get_helper.fire(get_valid)) - - when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { - w_inflight := Bool(true) - } - - when (w_inflight) { - when (tl_wrap_out) { w_inflight := Bool(false) } - } - - // Aggregate incoming NASTI responses into TL Grants - val (tl_cnt_in, tl_wrap_in) = Counter( - io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - gnt_arb.io.in(0).valid := io.nasti.r.valid - io.nasti.r.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - is_builtin_type = Bool(true), - g_type = Mux(roq.io.deq.data.subblock, - Grant.getDataBeatType, Grant.getDataBlockType), - client_xact_id = get_id_mapper.io.resp.tl_id, - manager_xact_id = UInt(0), - addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = io.nasti.r.bits.data) - - assert(!roq.io.deq.valid || roq.io.deq.matches, - "TL -> NASTI converter ReorderQueue: NASTI tag error") - assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, - "TL -> NASTI ID Mapper: NASTI tag error") - - gnt_arb.io.in(1).valid := io.nasti.b.valid - io.nasti.b.ready := gnt_arb.io.in(1).ready - gnt_arb.io.in(1).bits := Grant( - is_builtin_type = Bool(true), - g_type = Grant.putAckType, - client_xact_id = put_id_mapper.io.resp.tl_id, - manager_xact_id = UInt(0), - addr_beat = UInt(0), - data = Bits(0)) - assert(!gnt_arb.io.in(1).valid || put_id_mapper.io.resp.matches, "NASTI tag error") - - assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") - assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") -} - -class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) - with HasNastiParameters { - val io = new Bundle { - val nasti = (new NastiIO).flip - val tl = new ClientUncachedTileLinkIO - } - - val (s_idle :: s_put :: Nil) = Enum(Bits(), 2) - val state = Reg(init = s_idle) - - private val blockOffset = tlByteAddrBits + tlBeatAddrBits - - val aw_req = Reg(new NastiWriteAddressChannel) - - def is_singlebeat(chan: NastiAddressChannel): Bool = - chan.len === UInt(0) - - def is_multibeat(chan: NastiAddressChannel): Bool = - chan.len === UInt(tlDataBeats - 1) && chan.size === UInt(log2Up(tlDataBytes)) - - def nasti_addr_block(chan: NastiAddressChannel): UInt = - chan.addr(nastiXAddrBits - 1, blockOffset) - - def nasti_addr_beat(chan: NastiAddressChannel): UInt = - chan.addr(blockOffset - 1, tlByteAddrBits) - - def nasti_addr_byte(chan: NastiAddressChannel): UInt = - chan.addr(tlByteAddrBits - 1, 0) - - def nasti_operand_size(chan: NastiAddressChannel): UInt = - MuxLookup(chan.size, MT_Q, Seq( - UInt(0) -> MT_BU, - UInt(1) -> MT_HU, - UInt(2) -> MT_WU, - UInt(3) -> MT_D)) - - def size_mask(size: UInt): UInt = - (UInt(1) << (UInt(1) << size)) - UInt(1) - - def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { - val base = w.strb & size_mask(aw.size) - val addr_byte = nasti_addr_byte(aw) - w.strb & (size_mask(aw.size) << addr_byte) - } - - def tl_last(gnt: GrantMetadata): Bool = - !gnt.hasMultibeatData() || gnt.addr_beat === UInt(tlDataBeats - 1) - - def tl_b_grant(gnt: GrantMetadata): Bool = - gnt.g_type === Grant.putAckType - - assert(!io.nasti.ar.valid || - is_singlebeat(io.nasti.ar.bits) || is_multibeat(io.nasti.ar.bits), - "NASTI read transaction cannot convert to TileLInk") - - assert(!io.nasti.aw.valid || - is_singlebeat(io.nasti.aw.bits) || is_multibeat(io.nasti.aw.bits), - "NASTI write transaction cannot convert to TileLInk") - - val put_count = Reg(init = UInt(0, tlBeatAddrBits)) - - when (io.nasti.aw.fire()) { - aw_req := io.nasti.aw.bits - state := s_put - } - - when (io.nasti.w.fire()) { - put_count := put_count + UInt(1) - when (io.nasti.w.bits.last) { - put_count := UInt(0) - state := s_idle - } - } - - val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), - GetBlock( - client_xact_id = io.nasti.ar.bits.id, - addr_block = nasti_addr_block(io.nasti.ar.bits)), - Get( - client_xact_id = io.nasti.ar.bits.id, - addr_block = nasti_addr_block(io.nasti.ar.bits), - addr_beat = nasti_addr_beat(io.nasti.ar.bits), - addr_byte = nasti_addr_byte(io.nasti.ar.bits), - operand_size = nasti_operand_size(io.nasti.ar.bits), - alloc = Bool(false))) - - val put_acquire = Mux(is_multibeat(aw_req), - PutBlock( - client_xact_id = aw_req.id, - addr_block = nasti_addr_block(aw_req), - addr_beat = put_count, - data = io.nasti.w.bits.data, - wmask = io.nasti.w.bits.strb), - Put( - client_xact_id = aw_req.id, - addr_block = nasti_addr_block(aw_req), - addr_beat = nasti_addr_beat(aw_req), - data = io.nasti.w.bits.data, - wmask = nasti_wmask(aw_req, io.nasti.w.bits))) - - io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) - io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || - (state === s_put && io.nasti.w.valid) - io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) - io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) - io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) - - val nXacts = tlMaxClientXacts * tlMaxClientsPerPort - - io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) - io.nasti.b.bits := NastiWriteResponseChannel( - id = io.tl.grant.bits.client_xact_id) - - io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) - io.nasti.r.bits := NastiReadDataChannel( - id = io.tl.grant.bits.client_xact_id, - data = io.tl.grant.bits.data, - last = tl_last(io.tl.grant.bits)) - - io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), - io.nasti.b.ready, io.nasti.r.ready) -} - -object TileLinkWidthAdapter { - def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { - require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) - - if (out.tlDataBits > in.tlDataBits) { - val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) - widener.io.in <> in - out <> widener.io.out - } else if (out.tlDataBits < in.tlDataBits) { - val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) - narrower.io.in <> in - out <> narrower.io.out - } else { - out <> in - } - } -} - -class TileLinkIOWidener(innerTLId: String, outerTLId: String) - (implicit p: Parameters) extends TLModule()(p) { - - val paddrBits = p(PAddrBits) - val innerParams = p(TLKey(innerTLId)) - val outerParams = p(TLKey(outerTLId)) - val innerDataBeats = innerParams.dataBeats - val innerDataBits = innerParams.dataBitsPerBeat - val innerWriteMaskBits = innerParams.writeMaskBits - val innerByteAddrBits = log2Up(innerWriteMaskBits) - val innerMaxXacts = innerParams.maxClientXacts * innerParams.maxClientsPerPort - val innerXactIdBits = log2Up(innerMaxXacts) - val outerDataBeats = outerParams.dataBeats - val outerDataBits = outerParams.dataBitsPerBeat - val outerWriteMaskBits = outerParams.writeMaskBits - val outerByteAddrBits = log2Up(outerWriteMaskBits) - val outerBeatAddrBits = log2Up(outerDataBeats) - val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits - val outerMaxClients = outerParams.maxClientsPerPort - val outerClientIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - val outerManagerIdBits = log2Up(outerParams.maxManagerXacts) - val outerBlockAddrBits = paddrBits - outerBlockOffset - - require(outerDataBeats <= innerDataBeats) - require(outerDataBits >= innerDataBits) - require(outerDataBits % innerDataBits == 0) - require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) - - val factor = innerDataBeats / outerDataBeats - - val io = new Bundle { - val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip - val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) - } - - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits - val ignt = io.in.grant.bits - - val shrink = iacq.a_type === Acquire.putBlockType - val stretch = ognt.g_type === Grant.getDataBlockType - val smallget = iacq.a_type === Acquire.getType - val smallput = iacq.a_type === Acquire.putType - val smallgnt = ognt.g_type === Grant.getDataBeatType - - val sending_put = Reg(init = Bool(false)) - val collecting = Reg(init = Bool(false)) - val put_block = Reg(UInt(width = outerBlockAddrBits)) - val put_id = Reg(UInt(width = outerClientIdBits)) - val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) - val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) - val put_allocate = Reg(Bool()) - val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) - val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) - - val in_addr = iacq.full_addr() - val out_addr_block = in_addr(paddrBits - 1, outerBlockOffset) - val out_addr_beat = in_addr(outerBlockOffset - 1, outerByteAddrBits) - val out_addr_byte = in_addr(outerByteAddrBits - 1, 0) - - val switch_addr = in_addr(outerByteAddrBits - 1, innerByteAddrBits) - val smallget_switch = Reg(Vec(innerMaxXacts, switch_addr)) - - def align_data(addr: UInt, data: UInt): UInt = - data << Cat(addr, UInt(0, log2Up(innerDataBits))) - - def align_wmask(addr: UInt, wmask: UInt): UInt = - wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - addr_beat = out_addr_beat, - addr_byte = out_addr_byte, - operand_size = iacq.op_size(), - alloc = iacq.allocate()) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - alloc = iacq.allocate()) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - addr_beat = out_addr_beat, - data = align_data(switch_addr, iacq.data), - wmask = align_wmask(switch_addr, iacq.wmask()), - alloc = iacq.allocate()) - - val put_block_acquire = PutBlock( - client_xact_id = put_id, - addr_block = put_block, - addr_beat = put_beat, - data = put_data.toBits, - wmask = put_wmask.toBits) - - io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) - io.out.acquire.bits := MuxCase(get_block_acquire, Seq( - sending_put -> put_block_acquire, - smallget -> get_acquire, - smallput -> put_acquire)) - io.in.acquire.ready := !sending_put && (shrink || io.out.acquire.ready) - - when (io.in.acquire.fire() && shrink) { - when (!collecting) { - put_block := out_addr_block - put_id := iacq.client_xact_id - put_allocate := iacq.allocate() - collecting := Bool(true) - } - put_data(recv_idx) := iacq.data - put_wmask(recv_idx) := iacq.wmask() - } - - when (io.in.acquire.fire() && smallget) { - smallget_switch(iacq.client_xact_id) := switch_addr - } - - when (recv_done) { sending_put := Bool(true) } - when (sending_put && io.out.acquire.ready) { sending_put := Bool(false) } - when (put_done) { collecting := Bool(false) } - - val returning_data = Reg(init = Bool(false)) - val (send_idx, send_done) = Counter( - io.in.grant.ready && returning_data, factor) - - val gnt_beat = Reg(UInt(width = outerBeatAddrBits)) - val gnt_client_id = Reg(UInt(width = outerClientIdBits)) - val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) - val gnt_data = Reg(UInt(width = outerDataBits)) - - when (io.out.grant.fire() && stretch) { - gnt_data := ognt.data - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - gnt_beat := ognt.addr_beat - returning_data := Bool(true) - } - - when (send_done) { returning_data := Bool(false) } - - def select_data(data: UInt, sel: UInt): UInt = - data >> (sel << log2Up(innerDataBits)) - - val gnt_switch = smallget_switch(ognt.client_xact_id) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = Cat(gnt_beat, send_idx), - data = select_data(gnt_data, send_idx)) - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = Cat(ognt.addr_beat, gnt_switch), - data = select_data(ognt.data, gnt_switch)) - - val default_grant = Grant( - is_builtin_type = Bool(true), - g_type = ognt.g_type, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) - io.in.grant.bits := MuxCase(default_grant, Seq( - returning_data -> get_block_grant, - smallgnt -> get_grant)) - io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) -} - -class TileLinkIONarrower(innerTLId: String, outerTLId: String) - (implicit p: Parameters) extends TLModule()(p) { - - val innerParams = p(TLKey(innerTLId)) - val outerParams = p(TLKey(outerTLId)) - val innerDataBeats = innerParams.dataBeats - val innerDataBits = innerParams.dataBitsPerBeat - val innerWriteMaskBits = innerParams.writeMaskBits - val innerByteAddrBits = log2Up(innerWriteMaskBits) - val outerDataBeats = outerParams.dataBeats - val outerDataBits = outerParams.dataBitsPerBeat - val outerWriteMaskBits = outerParams.writeMaskBits - val outerByteAddrBits = log2Up(outerWriteMaskBits) - val outerBeatAddrBits = log2Up(outerDataBeats) - val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits - val outerMaxClients = outerParams.maxClientsPerPort - val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - - require(outerDataBeats > innerDataBeats) - require(outerDataBeats % innerDataBeats == 0) - require(outerDataBits < innerDataBits) - require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) - - val factor = outerDataBeats / innerDataBeats - - val io = new Bundle { - val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip - val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) - } - - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits - - val stretch = iacq.a_type === Acquire.putBlockType - val shrink = iacq.a_type === Acquire.getBlockType - val smallput = iacq.a_type === Acquire.putType - val smallget = iacq.a_type === Acquire.getType - - val acq_data_buffer = Reg(UInt(width = innerDataBits)) - val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) - val acq_client_id = Reg(iacq.client_xact_id) - val acq_addr_block = Reg(iacq.addr_block) - val acq_addr_beat = Reg(iacq.addr_beat) - val oacq_ctr = Counter(factor) - - val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) - val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) - - val mask_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerWriteMaskBits - val msb = (i + 1) * outerWriteMaskBits - 1 - iacq.wmask()(msb, lsb) - } - - val data_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerDataBits - val msb = (i + 1) * outerDataBits - 1 - iacq.data(msb, lsb) - } - - val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) - - val smallput_data = Mux1H(beat_sel, data_chunks) - val smallput_wmask = Mux1H(beat_sel, mask_chunks) - val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) - - assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), - "Can't perform Put wider than outer width") - - val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( - MT_B -> Bool(true), - MT_BU -> Bool(true), - MT_H -> Bool(outerDataBits >= 16), - MT_HU -> Bool(outerDataBits >= 16), - MT_W -> Bool(outerDataBits >= 32), - MT_WU -> Bool(outerDataBits >= 32), - MT_D -> Bool(outerDataBits >= 64), - MT_Q -> Bool(false))) - - assert(!io.in.acquire.valid || !smallget || read_size_ok, - "Can't perform Get wider than outer width") - - val outerConfig = p.alterPartial({ case TLId => outerTLId }) - val innerConfig = p.alterPartial({ case TLId => innerTLId }) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - alloc = iacq.allocate())(outerConfig) - - val put_block_acquire = PutBlock( - client_xact_id = acq_client_id, - addr_block = acq_addr_block, - addr_beat = if (factor > 1) - Cat(acq_addr_beat, oacq_ctr.value) - else acq_addr_beat, - data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = outer_beat_addr, - addr_byte = outer_byte_addr, - operand_size = iacq.op_size(), - alloc = iacq.allocate())(outerConfig) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = smallput_beat, - data = smallput_data, - wmask = Some(smallput_wmask))(outerConfig) - - val sending_put = Reg(init = Bool(false)) - - val pass_valid = io.in.acquire.valid && !stretch - - io.out.acquire.bits := MuxCase(Wire(io.out.acquire.bits, init=iacq), Seq( - (sending_put, put_block_acquire), - (shrink, get_block_acquire), - (smallput, put_acquire), - (smallget, get_acquire))) - io.out.acquire.valid := sending_put || pass_valid - io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) - - when (io.in.acquire.fire() && stretch) { - acq_data_buffer := iacq.data - acq_wmask_buffer := iacq.wmask() - acq_client_id := iacq.client_xact_id - acq_addr_block := iacq.addr_block - acq_addr_beat := iacq.addr_beat - sending_put := Bool(true) - } - - when (sending_put && io.out.acquire.ready) { - acq_data_buffer := acq_data_buffer >> outerDataBits - acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits - when (oacq_ctr.inc()) { sending_put := Bool(false) } - } - - val ognt_block = ognt.hasMultibeatData() - val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) - val gnt_client_id = Reg(ognt.client_xact_id) - val gnt_manager_id = Reg(ognt.manager_xact_id) - - val ignt_ctr = Counter(innerDataBeats) - val ognt_ctr = Counter(factor) - val sending_get = Reg(init = Bool(false)) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits)(innerConfig) - - val smallget_grant = ognt.g_type === Grant.getDataBeatType - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), - data = Fill(factor, ognt.data))(innerConfig) - - io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) - io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - - io.in.grant.bits := MuxCase(Wire(io.in.grant.bits, init=ognt), Seq( - sending_get -> get_block_grant, - smallget_grant -> get_grant)) - - when (io.out.grant.valid && ognt_block && !sending_get) { - gnt_data_buffer(ognt_ctr.value) := ognt.data - when (ognt_ctr.inc()) { - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - sending_get := Bool(true) - } - } - - when (io.in.grant.ready && sending_get) { - ignt_ctr.inc() - sending_get := Bool(false) - } -} - -class MMIOTileLinkManagerData(implicit p: Parameters) - extends TLBundle()(p) - with HasClientId - with HasClientTransactionId - -class MMIOTileLinkManager(implicit p: Parameters) - extends CoherenceAgentModule()(p) { - val io = new ManagerTLIO - - // MMIO requests should never need probe or release - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - - val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData() - val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0) - val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1) - - // Acquire and Grant are basically passthru, - // except client_id and client_xact_id need to be converted. - // Associate the inner client_id and client_xact_id - // with the outer client_xact_id. - val xact_pending = Reg(init = UInt(0, maxManagerXacts)) - val xact_id_sel = PriorityEncoder(~xact_pending) - val xact_id_reg = RegEnable(xact_id_sel, multibeat_start) - val xact_multibeat = Reg(init = Bool(false)) - val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel) - val xact_free = !xact_pending.andR - val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData)) - - io.inner.acquire.ready := io.outer.acquire.ready && xact_free - io.outer.acquire.valid := io.inner.acquire.valid && xact_free - io.outer.acquire.bits := io.inner.acquire.bits - io.outer.acquire.bits.client_xact_id := outer_xact_id - - def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool = - !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1) - - def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt = - Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0)) - - def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt = - ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(), - UIntToOH(in.bits.manager_xact_id), UInt(0)) - - def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt = - ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0)) - - xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) & - clearPendingBitOnFin(io.inner.finish) & - clearPendingBitOnGnt(io.inner.grant) - - when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) { - xact_buffer(outer_xact_id) := io.iacq() - } - - when (multibeat_start) { xact_multibeat := Bool(true) } - when (multibeat_end) { xact_multibeat := Bool(false) } - - val gnt_xact = xact_buffer(io.ognt().client_xact_id) - io.outer.grant.ready := io.inner.grant.ready - io.inner.grant.valid := io.outer.grant.valid - io.inner.grant.bits := io.outer.grant.bits - io.inner.grant.bits.client_id := gnt_xact.client_id - io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id - io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id - io.inner.finish.ready := Bool(true) -} diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/converters/Ahb.scala similarity index 99% rename from uncore/src/main/scala/ahb.scala rename to uncore/src/main/scala/converters/Ahb.scala index adee3f8a..0fca9517 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/converters/Ahb.scala @@ -1,7 +1,10 @@ -package uncore +package uncore.converters import Chisel._ import junctions._ +import uncore.tilelink._ +import uncore.util._ +import uncore.constants._ import cde.{Parameters, Field} import HastiConstants._ diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala new file mode 100644 index 00000000..006485ca --- /dev/null +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -0,0 +1,353 @@ +package uncore.converters + +import Chisel._ +import junctions._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters + +class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module + with HasTileLinkParameters with HasNastiParameters { + val io = new Bundle { + val req = new Bundle { + val valid = Bool(INPUT) + val ready = Bool(OUTPUT) + val tl_id = UInt(INPUT, tlClientXactIdBits) + val nasti_id = UInt(OUTPUT, nastiXIdBits) + } + val resp = new Bundle { + val valid = Bool(INPUT) + val matches = Bool(OUTPUT) + val nasti_id = UInt(INPUT, nastiXIdBits) + val tl_id = UInt(OUTPUT, tlClientXactIdBits) + } + } + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + + if (tlClientXactIdBits <= nastiXIdBits) { + io.req.ready := Bool(true) + io.req.nasti_id := io.req.tl_id + io.resp.matches := Bool(true) + io.resp.tl_id := io.resp.nasti_id + } else if (nastiXIdBits <= 2) { + val nQueues = 1 << nastiXIdBits + val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) + io.req.ready := Bool(false) + io.resp.matches := Bool(false) + io.resp.tl_id := UInt(0) + io.req.nasti_id := req_nasti_id + for (i <- 0 until nQueues) { + val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) + queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) + queue.io.enq.bits := io.req.tl_id + when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } + + queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) + when (io.resp.nasti_id === UInt(i)) { + io.resp.matches := queue.io.deq.valid + io.resp.tl_id := queue.io.deq.bits + } + } + } else { + val maxNastiId = 1 << nastiXIdBits + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) + val roq = Module(new ReorderQueue( + UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) + roq.io.enq.valid := io.req.valid + roq.io.enq.bits.data := io.req.tl_id + roq.io.enq.bits.tag := req_nasti_id + io.req.ready := roq.io.enq.ready + io.req.nasti_id := req_nasti_id + roq.io.deq.valid := io.resp.valid + roq.io.deq.tag := io.resp.nasti_id + io.resp.tl_id := roq.io.deq.data + io.resp.matches := roq.io.deq.matches + } +} + +class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { + val addr_beat = UInt(width = tlBeatAddrBits) + val subblock = Bool() +} + +class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val tl = new ClientUncachedTileLinkIO().flip + val nasti = new NastiIO + } + + private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( + MT_B -> UInt(0), + MT_BU -> UInt(0), + MT_H -> UInt(1), + MT_HU -> UInt(1), + MT_W -> UInt(2), + MT_WU -> UInt(2), + MT_D -> UInt(3), + MT_Q -> UInt(log2Up(tlDataBytes)))) + + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction + require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") + + val has_data = io.tl.acquire.bits.hasData() + + val is_subblock = io.tl.acquire.bits.isSubBlockType() + val is_multibeat = io.tl.acquire.bits.hasMultibeatData() + val (tl_cnt_out, tl_wrap_out) = Counter( + io.tl.acquire.fire() && is_multibeat, tlDataBeats) + + val get_valid = io.tl.acquire.valid && !has_data + val put_valid = io.tl.acquire.valid && has_data + + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + + // Reorder queue saves extra information needed to send correct + // grant back to TL client + val roq = Module(new ReorderQueue( + new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) + + val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + + val get_id_ready = get_id_mapper.io.req.ready + val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) + val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask + + // For Get/GetBlock, make sure Reorder queue can accept new entry + val get_helper = DecoupledHelper( + get_valid, + roq.io.enq.ready, + io.nasti.ar.ready, + get_id_ready) + + val w_inflight = Reg(init = Bool(false)) + + // For Put/PutBlock, make sure aw and w channel are both ready before + // we send the first beat + val aw_ready = w_inflight || io.nasti.aw.ready + val put_helper = DecoupledHelper( + put_valid, + aw_ready, + io.nasti.w.ready, + put_id_ready) + + val (nasti_cnt_out, nasti_wrap_out) = Counter( + io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) + + roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) + roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat + roq.io.enq.bits.data.subblock := is_subblock + roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) + roq.io.deq.tag := io.nasti.r.bits.id + + get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) + get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last + get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id + + put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) + put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + put_id_mapper.io.resp.valid := io.nasti.b.fire() + put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id + + // Decompose outgoing TL Acquires into Nasti address and data channels + io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) + io.nasti.ar.bits := NastiReadAddressChannel( + id = get_id_mapper.io.req.nasti_id, + addr = io.tl.acquire.bits.full_addr(), + size = Mux(is_subblock, + opSizeToXSize(io.tl.acquire.bits.op_size()), + UInt(log2Ceil(tlDataBytes))), + len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) + io.nasti.aw.bits := NastiWriteAddressChannel( + id = put_id_mapper.io.req.nasti_id, + addr = io.tl.acquire.bits.full_addr(), + size = UInt(log2Ceil(tlDataBytes)), + len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) + + io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) + io.nasti.w.bits := NastiWriteDataChannel( + id = put_id_mapper.io.req.nasti_id, + data = io.tl.acquire.bits.data, + strb = io.tl.acquire.bits.wmask(), + last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) + + io.tl.acquire.ready := Mux(has_data, + put_helper.fire(put_valid), + get_helper.fire(get_valid)) + + when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { + w_inflight := Bool(true) + } + + when (w_inflight) { + when (tl_wrap_out) { w_inflight := Bool(false) } + } + + // Aggregate incoming NASTI responses into TL Grants + val (tl_cnt_in, tl_wrap_in) = Counter( + io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + is_builtin_type = Bool(true), + g_type = Mux(roq.io.deq.data.subblock, + Grant.getDataBeatType, Grant.getDataBlockType), + client_xact_id = get_id_mapper.io.resp.tl_id, + manager_xact_id = UInt(0), + addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), + data = io.nasti.r.bits.data) + + assert(!roq.io.deq.valid || roq.io.deq.matches, + "TL -> NASTI converter ReorderQueue: NASTI tag error") + assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, + "TL -> NASTI ID Mapper: NASTI tag error") + + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).bits := Grant( + is_builtin_type = Bool(true), + g_type = Grant.putAckType, + client_xact_id = put_id_mapper.io.resp.tl_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = Bits(0)) + assert(!gnt_arb.io.in(1).valid || put_id_mapper.io.resp.matches, "NASTI tag error") + + assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") + assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") +} + +class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val nasti = (new NastiIO).flip + val tl = new ClientUncachedTileLinkIO + } + + val (s_idle :: s_put :: Nil) = Enum(Bits(), 2) + val state = Reg(init = s_idle) + + private val blockOffset = tlByteAddrBits + tlBeatAddrBits + + val aw_req = Reg(new NastiWriteAddressChannel) + + def is_singlebeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(0) + + def is_multibeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(tlDataBeats - 1) && chan.size === UInt(log2Up(tlDataBytes)) + + def nasti_addr_block(chan: NastiAddressChannel): UInt = + chan.addr(nastiXAddrBits - 1, blockOffset) + + def nasti_addr_beat(chan: NastiAddressChannel): UInt = + chan.addr(blockOffset - 1, tlByteAddrBits) + + def nasti_addr_byte(chan: NastiAddressChannel): UInt = + chan.addr(tlByteAddrBits - 1, 0) + + def nasti_operand_size(chan: NastiAddressChannel): UInt = + MuxLookup(chan.size, MT_Q, Seq( + UInt(0) -> MT_BU, + UInt(1) -> MT_HU, + UInt(2) -> MT_WU, + UInt(3) -> MT_D)) + + def size_mask(size: UInt): UInt = + (UInt(1) << (UInt(1) << size)) - UInt(1) + + def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { + val base = w.strb & size_mask(aw.size) + val addr_byte = nasti_addr_byte(aw) + w.strb & (size_mask(aw.size) << addr_byte) + } + + def tl_last(gnt: GrantMetadata): Bool = + !gnt.hasMultibeatData() || gnt.addr_beat === UInt(tlDataBeats - 1) + + def tl_b_grant(gnt: GrantMetadata): Bool = + gnt.g_type === Grant.putAckType + + assert(!io.nasti.ar.valid || + is_singlebeat(io.nasti.ar.bits) || is_multibeat(io.nasti.ar.bits), + "NASTI read transaction cannot convert to TileLInk") + + assert(!io.nasti.aw.valid || + is_singlebeat(io.nasti.aw.bits) || is_multibeat(io.nasti.aw.bits), + "NASTI write transaction cannot convert to TileLInk") + + val put_count = Reg(init = UInt(0, tlBeatAddrBits)) + + when (io.nasti.aw.fire()) { + aw_req := io.nasti.aw.bits + state := s_put + } + + when (io.nasti.w.fire()) { + put_count := put_count + UInt(1) + when (io.nasti.w.bits.last) { + put_count := UInt(0) + state := s_idle + } + } + + val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), + GetBlock( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits)), + Get( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits), + addr_beat = nasti_addr_beat(io.nasti.ar.bits), + addr_byte = nasti_addr_byte(io.nasti.ar.bits), + operand_size = nasti_operand_size(io.nasti.ar.bits), + alloc = Bool(false))) + + val put_acquire = Mux(is_multibeat(aw_req), + PutBlock( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = put_count, + data = io.nasti.w.bits.data, + wmask = io.nasti.w.bits.strb), + Put( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = nasti_addr_beat(aw_req), + data = io.nasti.w.bits.data, + wmask = Some(nasti_wmask(aw_req, io.nasti.w.bits)))) + + io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) + io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || + (state === s_put && io.nasti.w.valid) + io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) + io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) + io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) + + val nXacts = tlMaxClientXacts * tlMaxClientsPerPort + + io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) + io.nasti.b.bits := NastiWriteResponseChannel( + id = io.tl.grant.bits.client_xact_id) + + io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) + io.nasti.r.bits := NastiReadDataChannel( + id = io.tl.grant.bits.client_xact_id, + data = io.tl.grant.bits.data, + last = tl_last(io.tl.grant.bits)) + + io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), + io.nasti.b.ready, io.nasti.r.ready) +} + + diff --git a/uncore/src/main/scala/smi.scala b/uncore/src/main/scala/converters/Smi.scala similarity index 94% rename from uncore/src/main/scala/smi.scala rename to uncore/src/main/scala/converters/Smi.scala index dc8b158e..6ec47950 100644 --- a/uncore/src/main/scala/smi.scala +++ b/uncore/src/main/scala/converters/Smi.scala @@ -1,9 +1,10 @@ // See LICENSE for details -package uncore +package uncore.converters import Chisel._ import junctions._ +import uncore.tilelink._ import cde.Parameters /** Convert TileLink protocol to Smi protocol */ diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala new file mode 100644 index 00000000..4899f05d --- /dev/null +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -0,0 +1,538 @@ +package uncore.converters + +import Chisel._ +import junctions._ +import uncore.tilelink._ +import uncore.util._ +import uncore.constants._ +import cde.Parameters + +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ +object TileLinkIOWrapper { + def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientTileLinkIO().flip + val out = new ClientUncachedTileLinkIO + } + + val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, + Some((acq: Acquire) => acq.hasMultibeatData()))) + + val acqRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val relRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val iacq = io.in.acquire.bits + val irel = io.in.release.bits + val ognt = io.out.grant.bits + + val acq_roq_enq = iacq.first() + val rel_roq_enq = irel.first() + + val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready + val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready + + val acq_helper = DecoupledHelper( + io.in.acquire.valid, + acq_roq_ready, + acqArb.io.in(0).ready) + + val rel_helper = DecoupledHelper( + io.in.release.valid, + rel_roq_ready, + acqArb.io.in(1).ready) + + acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) + acqRoq.io.enq.bits.data := iacq.isBuiltInType() + acqRoq.io.enq.bits.tag := iacq.client_xact_id + + acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) + acqArb.io.in(0).bits := Acquire( + is_builtin_type = Bool(true), + a_type = Mux(iacq.isBuiltInType(), + iacq.a_type, Acquire.getBlockType), + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = iacq.addr_beat, + data = iacq.data, + union = Mux(iacq.isBuiltInType(), + iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) + io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) + + relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) + relRoq.io.enq.bits.data := irel.isVoluntary() + relRoq.io.enq.bits.tag := irel.client_xact_id + + acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) + acqArb.io.in(1).bits := PutBlock( + client_xact_id = irel.client_xact_id, + addr_block = irel.addr_block, + addr_beat = irel.addr_beat, + data = irel.data, + wmask = Acquire.fullWriteMask) + io.in.release.ready := rel_helper.fire(io.in.release.valid) + + io.out.acquire <> acqArb.io.out + + val grant_deq_roq = io.out.grant.fire() && ognt.last() + + acqRoq.io.deq.valid := acqRoq.io.deq.matches && grant_deq_roq + acqRoq.io.deq.tag := ognt.client_xact_id + + relRoq.io.deq.valid := !acqRoq.io.deq.matches && grant_deq_roq + relRoq.io.deq.tag := ognt.client_xact_id + + assert(!grant_deq_roq || acqRoq.io.deq.matches || relRoq.io.deq.matches, + "TileLink Unwrapper: client_xact_id mismatch") + + val gnt_builtin = acqRoq.io.deq.data + val gnt_voluntary = relRoq.io.deq.data + + val acq_grant = Grant( + is_builtin_type = gnt_builtin, + g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.") + val rel_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := io.out.grant.valid + io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) + io.out.grant.ready := io.in.grant.ready + + io.in.probe.valid := Bool(false) +} + +object TileLinkWidthAdapter { + def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { + require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) + + if (out.tlDataBits > in.tlDataBits) { + val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + widener.io.in <> in + out <> widener.io.out + } else if (out.tlDataBits < in.tlDataBits) { + val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) + narrower.io.in <> in + out <> narrower.io.out + } else { + out <> in + } + } +} + +class TileLinkIOWidener(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val paddrBits = p(PAddrBits) + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val innerMaxXacts = innerParams.maxClientXacts * innerParams.maxClientsPerPort + val innerXactIdBits = log2Up(innerMaxXacts) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerClientIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + val outerManagerIdBits = log2Up(outerParams.maxManagerXacts) + val outerBlockAddrBits = paddrBits - outerBlockOffset + + require(outerDataBeats <= innerDataBeats) + require(outerDataBits >= innerDataBits) + require(outerDataBits % innerDataBits == 0) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = innerDataBeats / outerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + val ignt = io.in.grant.bits + + val shrink = iacq.a_type === Acquire.putBlockType + val stretch = ognt.g_type === Grant.getDataBlockType + val smallget = iacq.a_type === Acquire.getType + val smallput = iacq.a_type === Acquire.putType + val smallgnt = ognt.g_type === Grant.getDataBeatType + + val sending_put = Reg(init = Bool(false)) + val collecting = Reg(init = Bool(false)) + val put_block = Reg(UInt(width = outerBlockAddrBits)) + val put_id = Reg(UInt(width = outerClientIdBits)) + val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) + val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) + val put_allocate = Reg(Bool()) + val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) + val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) + + val in_addr = iacq.full_addr() + val out_addr_block = in_addr(paddrBits - 1, outerBlockOffset) + val out_addr_beat = in_addr(outerBlockOffset - 1, outerByteAddrBits) + val out_addr_byte = in_addr(outerByteAddrBits - 1, 0) + + val switch_addr = in_addr(outerByteAddrBits - 1, innerByteAddrBits) + val smallget_switch = Reg(Vec(innerMaxXacts, switch_addr)) + + def align_data(addr: UInt, data: UInt): UInt = + data << Cat(addr, UInt(0, log2Up(innerDataBits))) + + def align_wmask(addr: UInt, wmask: UInt): UInt = + wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + addr_byte = out_addr_byte, + operand_size = iacq.op_size(), + alloc = iacq.allocate()) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + alloc = iacq.allocate()) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + data = align_data(switch_addr, iacq.data), + wmask = Some(align_wmask(switch_addr, iacq.wmask())), + alloc = iacq.allocate()) + + val put_block_acquire = PutBlock( + client_xact_id = put_id, + addr_block = put_block, + addr_beat = put_beat, + data = put_data.toBits, + wmask = put_wmask.toBits) + + io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) + io.out.acquire.bits := MuxCase(get_block_acquire, Seq( + sending_put -> put_block_acquire, + smallget -> get_acquire, + smallput -> put_acquire)) + io.in.acquire.ready := !sending_put && (shrink || io.out.acquire.ready) + + when (io.in.acquire.fire() && shrink) { + when (!collecting) { + put_block := out_addr_block + put_id := iacq.client_xact_id + put_allocate := iacq.allocate() + collecting := Bool(true) + } + put_data(recv_idx) := iacq.data + put_wmask(recv_idx) := iacq.wmask() + } + + when (io.in.acquire.fire() && smallget) { + smallget_switch(iacq.client_xact_id) := switch_addr + } + + when (recv_done) { sending_put := Bool(true) } + when (sending_put && io.out.acquire.ready) { sending_put := Bool(false) } + when (put_done) { collecting := Bool(false) } + + val returning_data = Reg(init = Bool(false)) + val (send_idx, send_done) = Counter( + io.in.grant.ready && returning_data, factor) + + val gnt_beat = Reg(UInt(width = outerBeatAddrBits)) + val gnt_client_id = Reg(UInt(width = outerClientIdBits)) + val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) + val gnt_data = Reg(UInt(width = outerDataBits)) + + when (io.out.grant.fire() && stretch) { + gnt_data := ognt.data + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + gnt_beat := ognt.addr_beat + returning_data := Bool(true) + } + + when (send_done) { returning_data := Bool(false) } + + def select_data(data: UInt, sel: UInt): UInt = + data >> (sel << log2Up(innerDataBits)) + + val gnt_switch = smallget_switch(ognt.client_xact_id) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = Cat(gnt_beat, send_idx), + data = select_data(gnt_data, send_idx)) + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = Cat(ognt.addr_beat, gnt_switch), + data = select_data(ognt.data, gnt_switch)) + + val default_grant = Grant( + is_builtin_type = Bool(true), + g_type = ognt.g_type, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) + io.in.grant.bits := MuxCase(default_grant, Seq( + returning_data -> get_block_grant, + smallgnt -> get_grant)) + io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) +} + +class TileLinkIONarrower(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + + require(outerDataBeats > innerDataBeats) + require(outerDataBeats % innerDataBeats == 0) + require(outerDataBits < innerDataBits) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = outerDataBeats / innerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType + val smallput = iacq.a_type === Acquire.putType + val smallget = iacq.a_type === Acquire.getType + + val acq_data_buffer = Reg(UInt(width = innerDataBits)) + val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) + + val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) + val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) + + val mask_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerWriteMaskBits + val msb = (i + 1) * outerWriteMaskBits - 1 + iacq.wmask()(msb, lsb) + } + + val data_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerDataBits + val msb = (i + 1) * outerDataBits - 1 + iacq.data(msb, lsb) + } + + val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) + + val smallput_data = Mux1H(beat_sel, data_chunks) + val smallput_wmask = Mux1H(beat_sel, mask_chunks) + val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) + + assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), + "Can't perform Put wider than outer width") + + val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( + MT_B -> Bool(true), + MT_BU -> Bool(true), + MT_H -> Bool(outerDataBits >= 16), + MT_HU -> Bool(outerDataBits >= 16), + MT_W -> Bool(outerDataBits >= 32), + MT_WU -> Bool(outerDataBits >= 32), + MT_D -> Bool(outerDataBits >= 64), + MT_Q -> Bool(false))) + + assert(!io.in.acquire.valid || !smallget || read_size_ok, + "Can't perform Get wider than outer width") + + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate())(outerConfig) + + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + addr_byte = outer_byte_addr, + operand_size = iacq.op_size(), + alloc = iacq.allocate())(outerConfig) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = smallput_beat, + data = smallput_data, + wmask = Some(smallput_wmask))(outerConfig) + + val sending_put = Reg(init = Bool(false)) + + val pass_valid = io.in.acquire.valid && !stretch + + io.out.acquire.bits := MuxCase(Wire(io.out.acquire.bits, init=iacq), Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire), + (smallput, put_acquire), + (smallget, get_acquire))) + io.out.acquire.valid := sending_put || pass_valid + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) + } + + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(innerDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits)(innerConfig) + + val smallget_grant = ognt.g_type === Grant.getDataBeatType + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), + data = Fill(factor, ognt.data))(innerConfig) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + + io.in.grant.bits := MuxCase(Wire(io.in.grant.bits, init=ognt), Seq( + sending_get -> get_block_grant, + smallget_grant -> get_grant)) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) + } + } + + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } +} diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/devices/Bram.scala similarity index 98% rename from uncore/src/main/scala/bram.scala rename to uncore/src/main/scala/devices/Bram.scala index 379dfa0d..1be27216 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/devices/Bram.scala @@ -1,8 +1,9 @@ -package uncore +package uncore.devices import Chisel._ import cde.{Parameters, Field} import junctions._ +import uncore.tilelink._ import HastiConstants._ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/devices/Debug.scala similarity index 99% rename from uncore/src/main/scala/debug.scala rename to uncore/src/main/scala/devices/Debug.scala index a065d442..9513e171 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/devices/Debug.scala @@ -1,9 +1,9 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ - +import uncore.tilelink._ import junctions._ import cde.{Parameters, Config, Field} diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/devices/Dma.scala similarity index 99% rename from uncore/src/main/scala/dma.scala rename to uncore/src/main/scala/devices/Dma.scala index fb157e07..eadff9d7 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/devices/Dma.scala @@ -1,9 +1,10 @@ -package uncore +package uncore.devices import Chisel._ import cde.{Parameters, Field} import junctions._ import junctions.NastiConstants._ +import uncore.tilelink._ case object NDmaTransactors extends Field[Int] case object NDmaXacts extends Field[Int] diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/devices/Plic.scala similarity index 99% rename from uncore/src/main/scala/plic.scala rename to uncore/src/main/scala/devices/Plic.scala index 7635e3aa..776c581b 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/devices/Plic.scala @@ -1,11 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ import Chisel.ImplicitConversions._ import junctions._ +import uncore.tilelink._ import cde.Parameters class GatewayPLICIO extends Bundle { diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/devices/Prci.scala similarity index 98% rename from uncore/src/main/scala/prci.scala rename to uncore/src/main/scala/devices/Prci.scala index 7f7db90d..19497aff 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/devices/Prci.scala @@ -1,11 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ import Chisel.ImplicitConversions._ import junctions._ import junctions.NastiConstants._ +import uncore.tilelink._ import cde.{Parameters, Field} /** Number of tiles */ diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/devices/Rom.scala similarity index 97% rename from uncore/src/main/scala/rom.scala rename to uncore/src/main/scala/devices/Rom.scala index 1ac56e1d..0fd9dd3e 100644 --- a/uncore/src/main/scala/rom.scala +++ b/uncore/src/main/scala/devices/Rom.scala @@ -1,7 +1,9 @@ -package uncore +package uncore.devices import Chisel._ import junctions._ +import uncore.tilelink._ +import uncore.util._ import cde.{Parameters, Field} class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala deleted file mode 100644 index 88f1a1b6..00000000 --- a/uncore/src/main/scala/network.scala +++ /dev/null @@ -1,122 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ -import cde.{Parameters, Field} - -case object LNEndpoints extends Field[Int] -case object LNHeaderBits extends Field[Int] - -class PhysicalHeader(n: Int) extends Bundle { - val src = UInt(width = log2Up(n)) - val dst = UInt(width = log2Up(n)) -} - -class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { - val header = new PhysicalHeader(n) - val payload = dType.cloneType - override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] -} - -class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { - val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip - val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) -} - -abstract class PhysicalNetwork extends Module - -case class CrossbarConfig[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) - -abstract class AbstractCrossbar[T <: Data](conf: CrossbarConfig[T]) extends PhysicalNetwork { - val io = new BasicCrossbarIO(conf.n, conf.dType) -} - -class BasicBus[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { - val arb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) - arb.io.in <> io.in - - arb.io.out.ready := io.out(arb.io.out.bits.header.dst).ready - for ((out, i) <- io.out zipWithIndex) { - out.valid := arb.io.out.valid && arb.io.out.bits.header.dst === UInt(i) - out.bits := arb.io.out.bits - } -} - -class BasicCrossbar[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { - io.in.foreach { _.ready := Bool(false) } - - io.out.zipWithIndex.map{ case (out, i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) - (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { - val destined = in.bits.header.dst === UInt(i) - arb.valid := in.valid && destined - arb.bits := in.bits - when (arb.ready && destined) { in.ready := Bool(true) } - }} - out <> rrarb.io.out - }} -} - -abstract class LogicalNetwork extends Module - -class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { - val src = UInt(width = p(LNHeaderBits)) - val dst = UInt(width = p(LNHeaderBits)) -} - -class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { - val header = new LogicalHeader - val payload = dType.cloneType - override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] -} - -object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data]( - in: DecoupledIO[T], - src: UInt = UInt(0), - dst: UInt = UInt(0)) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) - out.valid := in.valid - out.bits.payload := in.bits - out.bits.header.dst := dst - out.bits.header.src := src - in.ready := out.ready - out - } -} - -object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[T] = { - val out = Wire(Decoupled(in.bits.payload)) - out.valid := in.valid - out.bits := in.bits.payload - in.ready := out.ready - out - } -} - -object DefaultFromPhysicalShim { - def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) - out.bits.header := in.bits.header - out.bits.payload := in.bits.payload - out.valid := in.valid - in.ready := out.ready - out - } -} - -object DefaultToPhysicalShim { - def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { - val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) - out.bits.header := in.bits.header - out.bits.payload := in.bits.payload - out.valid := in.valid - in.ready := out.ready - out - } -} diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala deleted file mode 100644 index 2c6c4a5f..00000000 --- a/uncore/src/main/scala/package.scala +++ /dev/null @@ -1,6 +0,0 @@ -// See LICENSE for license details. - -package object uncore extends uncore.constants.MemoryOpConstants -{ - implicit def toOption[A](a: A) = Option(a) -} diff --git a/uncore/src/main/scala/tilelink/Arbiters.scala b/uncore/src/main/scala/tilelink/Arbiters.scala new file mode 100644 index 00000000..ab1f05ae --- /dev/null +++ b/uncore/src/main/scala/tilelink/Arbiters.scala @@ -0,0 +1,196 @@ +package uncore.tilelink +import Chisel._ +import junctions._ +import cde.{Parameters, Field} + +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends HasTileLinkParameters { + // Some shorthand type variables + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData + + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt + + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready + }} + mngr <> arb.io.out + } + + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + mngr <> arb.io.out + } + + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) + } + } + + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_&&_) + } + + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits, arbN)) + arb.io.in <> clts + mngr <> arb.io.out + } +} + +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ +abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec(arbN, new UncachedTileLinkIO).flip + val out = new UncachedTileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ +abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec(arbN, new TileLinkIO).flip + val out = new TileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Appends the port index of the arbiter to the client_xact_id */ +trait AppendsArbiterId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = { + /* This shouldn't be necessary, but Chisel3 doesn't emit correct Verilog + * when right shifting by too many bits. See + * https://github.com/ucb-bar/firrtl/issues/69 */ + if (in.client_xact_id.getWidth > log2Up(arbN)) + in.client_xact_id >> log2Up(arbN) + else + UInt(0) + } + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt +} + +/** Uses the client_xact_id as is (assumes it has been set to port index) */ +trait PassesId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +/** Overwrites some default client_xact_id with the port idx */ +trait UsesNewId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec(arbN, new ClientUncachedTileLinkIO).flip + val out = new ClientUncachedTileLinkIO + } + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec(arbN, new ClientTileLinkIO).flip + val out = new ClientTileLinkIO + } + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink/Definitions.scala similarity index 77% rename from uncore/src/main/scala/tilelink.scala rename to uncore/src/main/scala/tilelink/Definitions.scala index 5599217b..1917e403 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -1,11 +1,16 @@ // See LICENSE for license details. -package uncore +package uncore.tilelink import Chisel._ import junctions._ +import uncore.coherence.CoherencePolicy import scala.math.max +import uncore.constants._ import cde.{Parameters, Field} +case object CacheBlockOffsetBits extends Field[Int] +case object AmoAluOperandBits extends Field[Int] + case object TLId extends Field[String] case class TLKey(id: String) extends Field[TileLinkParameters] @@ -966,246 +971,3 @@ class ManagerTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val probe = new DecoupledIO(new ProbeToDst) val release = new DecoupledIO(new ReleaseFromSrc).flip } - -/** Struct for describing per-channel queue depths */ -case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) - -/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ -class TileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { - val io = new Bundle { - val client = new TileLinkIO().flip - val manager = new TileLinkIO - } - io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) - io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) - io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) - io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) - io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) -} - -object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): TileLinkIO = { - val t = Module(new TileLinkEnqueuer(depths)) - t.io.client <> in - t.io.manager - } - def apply(in: TileLinkIO, depth: Int)(implicit p: Parameters): TileLinkIO = { - apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) - } -} - -class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { - val io = new Bundle { - val inner = new ClientTileLinkIO().flip - val outer = new ClientTileLinkIO - } - - io.outer.acquire <> (if(depths.acq > 0) Queue(io.inner.acquire, depths.acq) else io.inner.acquire) - io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) - io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) - io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) - io.outer.finish <> (if(depths.fin > 0) Queue(io.inner.finish, depths.fin) else io.inner.finish) -} - -object ClientTileLinkEnqueuer { - def apply(in: ClientTileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): ClientTileLinkIO = { - val t = Module(new ClientTileLinkEnqueuer(depths)) - t.io.inner <> in - t.io.outer - } - def apply(in: ClientTileLinkIO, depth: Int)(implicit p: Parameters): ClientTileLinkIO = { - apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) - } -} - -/** Utility functions for constructing TileLinkIO arbiters */ -trait TileLinkArbiterLike extends HasTileLinkParameters { - // Some shorthand type variables - type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId - type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData - - val arbN: Int // The number of ports on the client side - - // These abstract funcs are filled in depending on whether the arbiter mucks with the - // outgoing client ids to track sourcing and then needs to revert them on the way back - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits - def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits - def arbIdx(in: ManagerSourcedWithId): UInt - - // The following functions are all wiring helpers for each of the different types of TileLink channels - - def hookupClientSource[M <: ClientSourcedWithIdAndData]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) - clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { - arb.valid := req.valid - arb.bits := req.bits - arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) - req.ready := arb.ready - }} - mngr <> arb.io.out - } - - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) - clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { - arb.valid := req.valid - arb.bits := req.bits - arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) - req.ready := arb.ready - }} - mngr <> arb.io.out - } - - def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (mngr.bits.header.dst === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - } - } - - def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (arbIdx(mngr.bits.payload) === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) - } - } - - def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (arbIdx(mngr.bits) === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) - } - } - - def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { - clts.map{ _.valid := mngr.valid } - clts.map{ _.bits := mngr.bits } - mngr.ready := clts.map(_.ready).reduce(_&&_) - } - - def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits, arbN)) - arb.io.in <> clts - mngr <> arb.io.out - } -} - -/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ -abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module - with TileLinkArbiterLike { - val io = new Bundle { - val in = Vec(arbN, new UncachedTileLinkIO).flip - val out = new UncachedTileLinkIO - } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupFinish(io.in.map(_.finish), io.out.finish) - hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) -} - -/** Abstract base case for any Arbiters that have cached TileLinkIOs */ -abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module - with TileLinkArbiterLike { - val io = new Bundle { - val in = Vec(arbN, new TileLinkIO).flip - val out = new TileLinkIO - } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupClientSource(io.in.map(_.release), io.out.release) - hookupFinish(io.in.map(_.finish), io.out.finish) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) -} - -/** Appends the port index of the arbiter to the client_xact_id */ -trait AppendsArbiterId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = - Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerSourcedWithId) = { - /* This shouldn't be necessary, but Chisel3 doesn't emit correct Verilog - * when right shifting by too many bits. See - * https://github.com/ucb-bar/firrtl/issues/69 */ - if (in.client_xact_id.getWidth > log2Up(arbN)) - in.client_xact_id >> log2Up(arbN) - else - UInt(0) - } - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt -} - -/** Uses the client_xact_id as is (assumes it has been set to port index) */ -trait PassesId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id - def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id -} - -/** Overwrites some default client_xact_id with the port idx */ -trait UsesNewId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id -} - -// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with UsesNewId - -/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { - val io = new Bundle { - val in = Vec(arbN, new ClientUncachedTileLinkIO).flip - val out = new ClientUncachedTileLinkIO - } - if (arbN > 1) { - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) - } else { io.out <> io.in.head } -} - -/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { - val io = new Bundle { - val in = Vec(arbN, new ClientTileLinkIO).flip - val out = new ClientTileLinkIO - } - if (arbN > 1) { - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) - } else { io.out <> io.in.head } -} diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/tilelink/Interconnect.scala similarity index 99% rename from uncore/src/main/scala/interconnect.scala rename to uncore/src/main/scala/tilelink/Interconnect.scala index eb9773a1..353dbb80 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/tilelink/Interconnect.scala @@ -1,8 +1,9 @@ -package uncore +package uncore.tilelink import Chisel._ import junctions._ import scala.collection.mutable.ArraySeq +import uncore.util._ import cde.{Parameters, Field} diff --git a/uncore/src/main/scala/tilelink/Network.scala b/uncore/src/main/scala/tilelink/Network.scala new file mode 100644 index 00000000..1c094013 --- /dev/null +++ b/uncore/src/main/scala/tilelink/Network.scala @@ -0,0 +1,308 @@ +// See LICENSE for license details. + +package uncore.tilelink + +import Chisel._ +import uncore.util._ +import cde.{Parameters, Field} + +case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] + +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) +} + +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) + val payload = dType.cloneType + override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] +} + +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip + val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) +} + +abstract class PhysicalNetwork extends Module + +case class CrossbarConfig[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) + +abstract class AbstractCrossbar[T <: Data](conf: CrossbarConfig[T]) extends PhysicalNetwork { + val io = new BasicCrossbarIO(conf.n, conf.dType) +} + +class BasicBus[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { + val arb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) + arb.io.in <> io.in + + arb.io.out.ready := io.out(arb.io.out.bits.header.dst).ready + for ((out, i) <- io.out zipWithIndex) { + out.valid := arb.io.out.valid && arb.io.out.bits.header.dst === UInt(i) + out.bits := arb.io.out.bits + } +} + +class BasicCrossbar[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { + io.in.foreach { _.ready := Bool(false) } + + io.out.zipWithIndex.map{ case (out, i) => { + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) + (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { + val destined = in.bits.header.dst === UInt(i) + arb.valid := in.valid && destined + arb.bits := in.bits + when (arb.ready && destined) { in.ready := Bool(true) } + }} + out <> rrarb.io.out + }} +} + +abstract class LogicalNetwork extends Module + +class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { + val src = UInt(width = p(LNHeaderBits)) + val dst = UInt(width = p(LNHeaderBits)) +} + +class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { + val header = new LogicalHeader + val payload = dType.cloneType + override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] +} + +object DecoupledLogicalNetworkIOWrapper { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out + } +} + +object DecoupledLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[T] = { + val out = Wire(Decoupled(in.bits.payload)) + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out + } +} + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ +class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + } + + val g = io.grant.bits.payload + + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounterWithHeader(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits := g.makeFinish() + q.io.enq.bits.manager_id := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.manager_id + io.finish.bits.payload := q.io.deq.bits + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } +} + +class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries) + +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probe Probes]]. + * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response, + * setting up the headers for each. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientTileLinkIO().flip + val network = new TileLinkIO + } + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) + val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant) + + io.network.acquire <> acq_with_header + io.network.release <> rel_with_header + io.network.finish <> fin_with_header + io.client.probe <> prb_without_header + io.client.grant.bits.manager_id := io.network.grant.bits.header.src + io.client.grant <> gnt_without_header +} + +/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientUncachedTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.client.grant <> gnt_without_header + io.network.probe.ready := Bool(false) + io.network.release.valid := Bool(false) +} + +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasManagerId]( + in: DecoupledIO[T], + clientId: Int) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := in.bits.manager_id + out.valid := in.valid + in.ready := out.ready + out + } + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip + } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) +} + +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/util/AmoAlu.scala similarity index 97% rename from uncore/src/main/scala/amoalu.scala rename to uncore/src/main/scala/util/AmoAlu.scala index 6459d558..d6ff9ce8 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/util/AmoAlu.scala @@ -1,8 +1,11 @@ // See LICENSE for license details. -package uncore +package uncore.util + import Chisel._ -import cde.{Parameters, Field} +import uncore.tilelink._ +import cde.Parameters +import uncore.constants._ class StoreGen(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) { val size = typ(log2Up(log2Up(maxSize)+1)-1,0) diff --git a/uncore/src/main/scala/util/Counters.scala b/uncore/src/main/scala/util/Counters.scala new file mode 100644 index 00000000..3bc2d85b --- /dev/null +++ b/uncore/src/main/scala/util/Counters.scala @@ -0,0 +1,134 @@ +package uncore.util + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +object TwoWayCounter { + def apply(up: Bool, down: Bool, max: Int): UInt = { + val cnt = Reg(init = UInt(0, log2Up(max+1))) + when (up && !down) { cnt := cnt + UInt(1) } + when (down && !up) { cnt := cnt - UInt(1) } + cnt + } +} + +class BeatCounterStatus extends Bundle { + val idx = UInt() + val done = Bool() +} + +class TwoWayBeatCounterStatus extends Bundle { + val pending = Bool() + val up = new BeatCounterStatus() + val down = new BeatCounterStatus() +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. + * + * Won't count message types that don't have data. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasId = TileLinkChannel with HasClientId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( + out: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(out.fire(), out.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param status bundle of status of the counters + * @param up outgoing channel + * @param down incoming channel + * @param max max number of outstanding ups with no down + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel]( + status: TwoWayBeatCounterStatus, + up: DecoupledIO[T], + down: DecoupledIO[S], + max: Int = 1, + beat: UInt = UInt(0), + trackUp: T => Bool = (t: T) => Bool(true), + trackDown: S => Bool = (s: S) => Bool(true)) { + val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat) + val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat) + val cnt = TwoWayCounter(up_done, dn_done, max) + status.pending := cnt > UInt(0) + status.up.idx := up_idx + status.up.done := up_done + status.down.idx := dn_idx + status.down.done := dn_done + } +} + + diff --git a/uncore/src/main/scala/util/Enqueuer.scala b/uncore/src/main/scala/util/Enqueuer.scala new file mode 100644 index 00000000..f6f3587a --- /dev/null +++ b/uncore/src/main/scala/util/Enqueuer.scala @@ -0,0 +1,56 @@ +package uncore.util + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +/** Struct for describing per-channel queue depths */ +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ +class TileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths)) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(implicit p: Parameters): TileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) + } +} + +class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { + val io = new Bundle { + val inner = new ClientTileLinkIO().flip + val outer = new ClientTileLinkIO + } + + io.outer.acquire <> (if(depths.acq > 0) Queue(io.inner.acquire, depths.acq) else io.inner.acquire) + io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) + io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) + io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) + io.outer.finish <> (if(depths.fin > 0) Queue(io.inner.finish, depths.fin) else io.inner.finish) +} + +object ClientTileLinkEnqueuer { + def apply(in: ClientTileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): ClientTileLinkIO = { + val t = Module(new ClientTileLinkEnqueuer(depths)) + t.io.inner <> in + t.io.outer + } + def apply(in: ClientTileLinkIO, depth: Int)(implicit p: Parameters): ClientTileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) + } +} diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util/Serializer.scala similarity index 69% rename from uncore/src/main/scala/util.scala rename to uncore/src/main/scala/util/Serializer.scala index b6d11fd5..8cc0caa2 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util/Serializer.scala @@ -1,40 +1,9 @@ // See LICENSE for license details. -package uncore +package uncore.util import Chisel._ - -// Produces 0-width value when counting to 1 -class ZCounter(val n: Int) { - val value = Reg(init=UInt(0, log2Ceil(n))) - def inc(): Bool = { - if (n == 1) Bool(true) - else { - val wrap = value === UInt(n-1) - value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) - wrap - } - } -} - -object ZCounter { - def apply(n: Int) = new ZCounter(n) - def apply(cond: Bool, n: Int): (UInt, Bool) = { - val c = new ZCounter(n) - var wrap: Bool = null - when (cond) { wrap = c.inc() } - (c.value, cond && wrap) - } -} - -object TwoWayCounter { - def apply(up: Bool, down: Bool, max: Int): UInt = { - val cnt = Reg(init = UInt(0, log2Up(max+1))) - when (up && !down) { cnt := cnt + UInt(1) } - when (down && !up) { cnt := cnt - UInt(1) } - cnt - } -} +import uncore.tilelink._ class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle {