From 10309849b7b7080b91281bf3cb5c95d5b5680900 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 5 Nov 2014 14:21:38 -0800 Subject: [PATCH 1/4] Remove master_xact_id from Probe and Release --- uncore/src/main/scala/tilelink.scala | 35 ++++++++++------------------ uncore/src/main/scala/uncore.scala | 8 +++---- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..6738deaf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -79,52 +79,41 @@ class Acquire extends ClientSourcedMessage val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } -object Probe -{ - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { + +object Probe { + def apply(p_type: UInt, addr: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr - prb.master_xact_id := master_xact_id prb } } class Probe extends MasterSourcedMessage - with HasPhysicalAddress - with HasMasterTransactionId { + with HasPhysicalAddress { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } -object Release -{ - def apply(r_type: UInt, addr: UInt, data: UInt): Release = { - val rel = new Release - rel.r_type := r_type - rel.addr := addr - rel.data := data - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { +object Release { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.client_xact_id := client_xact_id - rel.master_xact_id := master_xact_id - rel.data := UInt(0) - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { - val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { + apply(r_type, addr, client_xact_id, UInt(0)) + } + def apply(r_type: UInt, addr: UInt): Release = { + apply(r_type, addr, UInt(0), UInt(0)) + } } class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId - with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 37e535f8..e1f59e08 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -62,8 +62,8 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), conflict_idx) + // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -200,9 +200,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) From 35553cc0b71ffe63b1ea0b9d775e691582376db5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 16:05:25 -0800 Subject: [PATCH 2/4] NullDirectory sharers.count fix --- uncore/src/main/scala/coherence.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..9f94583a 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -61,14 +60,13 @@ abstract class DirectoryRepresentation extends Bundle { def next(dummy: Int = 0): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) +class NullRepresentation(nClients: Int) extends DirectoryRepresentation { def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) + def count(dummy: Int = 0) = UInt(nClients) def next(dummy: Int = 0) = UInt(0) } From 82155f333e96bfb5eaa3b3b25ff7184609e39a17 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 17:36:55 -0800 Subject: [PATCH 3/4] Major tilelink revision for uncached message types --- uncore/src/main/scala/coherence.scala | 566 ++++++++++---------------- uncore/src/main/scala/consts.scala | 2 + uncore/src/main/scala/htif.scala | 6 +- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 179 +++++--- uncore/src/main/scala/uncore.scala | 31 +- 6 files changed, 351 insertions(+), 435 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9f94583a..a2af20cb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -120,59 +120,48 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt - def messageHasData (rel: SourcedMessage): Bool - def messageUpdatesDataArray (reply: Grant): Bool - def messageIsUncached(acq: Acquire): Bool + def messageHasData(rel: SourcedMessage): Bool + def messageUpdatesDataArray(g: Grant): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterRead(a_type: UInt): Bool - def requiresOuterWrite(a_type: UInt): Bool - def requiresSelfProbe(a_type: UInt): Bool - def requiresAckForGrant(g_type: UInt): Bool - def requiresAckForRelease(r_type: UInt): Bool + def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresSelfProbe(a: Acquire): Bool + def requiresAckForGrant(g: Grant): Bool + def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) + } } -trait UncachedTransactions { - def getUncachedReadAcquireType: UInt - def getUncachedWriteAcquireType: UInt - def getUncachedReadWordAcquireType: UInt - def getUncachedWriteWordAcquireType: UInt - def getUncachedAtomicAcquireType: UInt - def isUncachedReadTransaction(acq: Acquire): Bool -} - -abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) - with UncachedTransactions - -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 - def nAcquireTypes = 6 + def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 - def nGrantTypes = 7 + def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid @@ -197,15 +186,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit M_CLN -> clientValid )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> clientValid, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -225,14 +207,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive @@ -251,80 +227,61 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } + def getGrantType(a: Acquire, m: MasterMetadata): UInt = + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 8 + def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid @@ -332,7 +289,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -355,16 +312,9 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -387,14 +337,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) @@ -419,83 +363,62 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusiveClean -> grantReadExclusive, - acquireReadExclusiveDirty -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusiveClean -> probeInvalidate, - acquireReadExclusiveDirty -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), @@ -508,7 +431,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -530,16 +453,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> clientExclusiveDirty, - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(incoming.g_type === grantReadShared, clientShared, + clientExclusiveDirty)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -564,14 +480,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -596,79 +506,69 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeInvalidate, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), @@ -681,7 +581,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -704,16 +604,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -738,14 +635,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -770,83 +661,68 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 3 - def nAcquireTypes = 8 + def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 - def nGrantTypes = 9 + def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, m: ClientMetadata): Bool = { @@ -859,7 +735,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -883,22 +759,18 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -935,14 +807,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) @@ -969,60 +835,50 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + MuxLookup(a.a_type, grantReadShared, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? + ))) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index a402e7db..fe647d07 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,9 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") + val MT_CB = Bits("b111") // cache block + val NUM_XA_OPS = 9 val M_SZ = 5 val M_X = Bits("b?????"); val M_XRD = Bits("b00000"); // int load diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index edcd66cb..405380f2 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -135,7 +135,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } io.mem.grant.ready := Bool(true) @@ -193,8 +193,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), - Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + UncachedWrite(init_addr, UInt(0)), + UncachedRead(init_addr)) io.mem.acquire.valid := acq_q.io.deq.valid acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..02e651b8 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(Bool(true), UInt(0), tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6738deaf..3b1bf26b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -2,65 +2,89 @@ package uncore import Chisel._ +import scala.math.max case object TLId extends Field[String] -case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] -case object TLWriteMaskBits extends Field[Int] -case object TLWordAddrBits extends Field[Int] -case object TLAtomicOpBits extends Field[Int] -trait HasPhysicalAddress extends Bundle { - val addr = UInt(width = params(TLAddrBits)) +abstract trait TileLinkParameters extends UsesParameters { + val tlAddrBits = params(TLAddrBits) + val tlClientXactIdBits = params(TLClientXactIdBits) + val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlDataBits = params(TLDataBits) + val tlWriteMaskBits = tlDataBits/8 + val tlSubblockAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlUncachedOperandSizeBits = MT_SZ + val tlSubblockUnionBits = max(tlWriteMaskBits, + (tlSubblockAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = params(TLClientXactIdBits)) +class TLBundle extends Bundle with TileLinkParameters + +trait HasPhysicalAddress extends TLBundle { + val addr = UInt(width = tlAddrBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = params(TLMasterXactIdBits)) +trait HasClientTransactionId extends TLBundle { + val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasTileLinkData extends Bundle { - val data = Bits(width = params(TLDataBits)) +trait HasMasterTransactionId extends TLBundle { + val master_xact_id = Bits(width = tlMasterXactIdBits) } -trait SourcedMessage extends Bundle +trait HasTileLinkData extends TLBundle { + val data = UInt(width = tlDataBits) +} + +trait SourcedMessage extends TLBundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -object Acquire -{ - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { +class Acquire extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val uncached = Bool() + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val subblock = Bits(width = tlSubblockUnionBits) + val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits + val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) + def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) +} + +object Acquire { + val nUncachedAcquireTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite + def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = new Acquire + acq.uncached := Bool(false) acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.data := Bits(0) - acq.write_mask := Bits(0) - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) - acq - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id) acq.data := data + acq.subblock := UInt(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.write_mask := write_mask - acq - } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.subword_addr := subword_addr - acq.atomic_opcode := atomic_opcode - acq + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { + apply(a_type, addr, client_xact_id, UInt(0)) } def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -69,16 +93,45 @@ object Acquire } } -class Acquire extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) - val write_mask = Bits(width = params(TLWriteMaskBits)) - val subword_addr = Bits(width = params(TLWordAddrBits)) - val atomic_opcode = Bits(width = params(TLAtomicOpBits)) +object UncachedRead { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) + acq.uncached := Bool(true) + acq.subblock := Cat(subblock_addr, operand_sz) + acq + } + def apply(addr: UInt, client_xact_id: UInt): Acquire = { + apply(addr, client_xact_id, UInt(0), MT_CB) + } + def apply(addr: UInt): Acquire = { + apply(addr, UInt(0), UInt(0), MT_CB) + } } +object UncachedWrite { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := write_mask + acq + } + def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(addr, client_xact_id, SInt(-1), data) + } + def apply(addr: UInt, data: UInt): Acquire = { + apply(addr, UInt(0), data) + } +} + +object UncachedAtomic { + def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, + subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq + } +} object Probe { def apply(p_type: UInt, addr: UInt) = { @@ -118,28 +171,34 @@ class Release extends ClientSourcedMessage val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } -object Grant -{ - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - val gnt = new Grant - gnt.g_type := g_type - gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id - gnt.data := UInt(0) - gnt - } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { - val gnt = apply(g_type, client_xact_id, master_xact_id) - gnt.data := data - gnt - } -} - class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + val uncached = Bool() + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) +} + +object Grant { + val nUncachedGrantTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + val gnt = new Grant + gnt.uncached := uncached + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := data + gnt + } + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + } } class Finish extends ClientSourcedMessage with HasMasterTransactionId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e1f59e08..6ea8cd25 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -122,7 +122,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), xact.data), @@ -133,7 +133,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), + io.inner.grant.bits.payload := Grant(Bool(false), + co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -169,15 +170,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) + val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -202,11 +201,11 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.bits.header.dst := curr_p_id io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, + io.inner.grant.bits.payload := Grant(xact.uncached, + co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) @@ -218,7 +217,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src @@ -266,7 +265,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_mem_write) { @@ -279,7 +278,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_make_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete From cb7e7125993c001c08d5159aa1e4d8bda3837c00 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Nov 2014 12:55:07 -0800 Subject: [PATCH 4/4] Added uncached write data queue to coherence hub --- uncore/src/main/scala/tilelink.scala | 2 +- uncore/src/main/scala/uncore.scala | 59 +++++++++++++++++++++------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b1bf26b..ec4c15df 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,7 +16,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) - val tlWriteMaskBits = tlDataBits/8 + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) val tlUncachedOperandSizeBits = MT_SZ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6ea8cd25..858d603c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -5,14 +5,17 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) - val nReleaseTransactors = params(NReleaseTransactors) + val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) + val sdqDepth = params(L2StoreDataQueueDepth) + val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { + // Queue to store impending UncachedWrite data + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} + when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId))) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -67,12 +79,19 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } + when(voluntary && release.fire()) { + vwbdq(release_idx) := release.bits.payload.data + } + // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) + { case TLId => outerId; case TLDataBits => sdqIdBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) + val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) + val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) + val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) + val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq + io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), + Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } } @@ -112,10 +144,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -140,11 +171,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute switch (state) { is(s_idle) { - io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + io.inner.release.ready := Bool(true) + xact := c_rel.payload + init_client_id := c_rel.header.src + state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) } } is(s_mem) { @@ -173,7 +204,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) @@ -208,7 +239,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + UInt(0)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false)