added support for voluntary wbs over the release network
This commit is contained in:
Henry Cook
parent
b5ccdab514
commit
47a632cc59
@ -31,6 +31,7 @@ abstract class CoherencePolicy {
|
||||
def getAcquireTypeOnCacheControl(cmd: Bits): Bits
|
||||
def getAcquireTypeOnWriteback(): Bits
|
||||
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release
|
||||
def newRelease (incoming: Probe, state: UFix): Release
|
||||
|
||||
def messageHasData (reply: Release): Bool
|
||||
@ -40,11 +41,15 @@ abstract class CoherencePolicy {
|
||||
def messageIsUncached(acq: Acquire): Bool
|
||||
|
||||
def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool
|
||||
def isVoluntary(rel: Release): Bool
|
||||
def getGrantType(a_type: UFix, count: UFix): Bits
|
||||
def getGrantType(rel: Release, count: UFix): Bits
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix
|
||||
def needsMemRead(a_type: UFix, global_state: UFix): Bool
|
||||
def needsMemWrite(a_type: UFix, global_state: UFix): Bool
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool
|
||||
def requiresAck(grant: Grant): Bool
|
||||
def requiresAck(release: Release): Bool
|
||||
}
|
||||
|
||||
trait UncachedTransactions {
|
||||
@ -70,17 +75,20 @@ abstract class IncoherentPolicy extends CoherencePolicy {
|
||||
def messageHasData (reply: Release) = Bool(false)
|
||||
def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false)
|
||||
def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0)
|
||||
def getGrantType(rel: Release, count: UFix): Bits = Bits(0)
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0)
|
||||
def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false)
|
||||
def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false)
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false)
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
class ThreeStateIncoherence extends IncoherentPolicy {
|
||||
val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() }
|
||||
val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() }
|
||||
val grantData :: grantAck :: Nil = Enum(2){ UFix() }
|
||||
val releaseInvalidateAck :: Nil = Enum(1){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() }
|
||||
val uncachedTypeList = List()
|
||||
val hasDataTypeList = List(acquireWriteback)
|
||||
|
||||
@ -106,6 +114,9 @@ class ThreeStateIncoherence extends IncoherentPolicy {
|
||||
))
|
||||
}
|
||||
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
|
||||
val (read, write) = cpuCmdToRW(cmd)
|
||||
Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean)
|
||||
@ -129,9 +140,9 @@ class MICoherence extends CoherencePolicyWithUncached {
|
||||
val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() }
|
||||
|
||||
val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() }
|
||||
val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() }
|
||||
val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() }
|
||||
val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() }
|
||||
val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() }
|
||||
val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
|
||||
@ -180,7 +191,9 @@ class MICoherence extends CoherencePolicyWithUncached {
|
||||
def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id)
|
||||
def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask)
|
||||
def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op)
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive
|
||||
def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive
|
||||
@ -228,6 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached {
|
||||
))
|
||||
}
|
||||
|
||||
def getGrantType(rel: Release, count: UFix): Bits = {
|
||||
MuxLookup(rel.r_type, grantReadUncached, Array(
|
||||
releaseVoluntaryInvalidateData -> grantVoluntaryAck
|
||||
))
|
||||
}
|
||||
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = {
|
||||
MuxLookup(a_type, probeCopy, Array(
|
||||
acquireReadExclusive -> probeInvalidate,
|
||||
@ -248,6 +267,8 @@ class MICoherence extends CoherencePolicyWithUncached {
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = {
|
||||
(a_type === acquireWriteUncached)
|
||||
}
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
class MEICoherence extends CoherencePolicyWithUncached {
|
||||
@ -256,9 +277,9 @@ class MEICoherence extends CoherencePolicyWithUncached {
|
||||
val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() }
|
||||
|
||||
val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() }
|
||||
val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() }
|
||||
val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() }
|
||||
val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() }
|
||||
val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() }
|
||||
val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
|
||||
@ -316,7 +337,9 @@ class MEICoherence extends CoherencePolicyWithUncached {
|
||||
def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id)
|
||||
def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask)
|
||||
def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op)
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
|
||||
val (read, write) = cpuCmdToRW(cmd)
|
||||
@ -373,6 +396,12 @@ class MEICoherence extends CoherencePolicyWithUncached {
|
||||
acquireAtomicUncached -> grantAtomicUncached
|
||||
))
|
||||
}
|
||||
def getGrantType(rel: Release, count: UFix): Bits = {
|
||||
MuxLookup(rel.r_type, grantReadUncached, Array(
|
||||
releaseVoluntaryInvalidateData -> grantVoluntaryAck
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = {
|
||||
MuxLookup(a_type, probeCopy, Array(
|
||||
@ -395,6 +424,8 @@ class MEICoherence extends CoherencePolicyWithUncached {
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = {
|
||||
(a_type === acquireWriteUncached)
|
||||
}
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
class MSICoherence extends CoherencePolicyWithUncached {
|
||||
@ -403,9 +434,9 @@ class MSICoherence extends CoherencePolicyWithUncached {
|
||||
val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() }
|
||||
|
||||
val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() }
|
||||
val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() }
|
||||
val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() }
|
||||
val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() }
|
||||
val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() }
|
||||
val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
|
||||
@ -470,7 +501,9 @@ class MSICoherence extends CoherencePolicyWithUncached {
|
||||
def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id)
|
||||
def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask)
|
||||
def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op)
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
|
||||
val (read, write) = cpuCmdToRW(cmd)
|
||||
@ -527,6 +560,12 @@ class MSICoherence extends CoherencePolicyWithUncached {
|
||||
acquireAtomicUncached -> grantAtomicUncached
|
||||
))
|
||||
}
|
||||
def getGrantType(rel: Release, count: UFix): Bits = {
|
||||
MuxLookup(rel.r_type, grantReadUncached, Array(
|
||||
releaseVoluntaryInvalidateData -> grantVoluntaryAck
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = {
|
||||
MuxLookup(a_type, probeCopy, Array(
|
||||
@ -546,6 +585,8 @@ class MSICoherence extends CoherencePolicyWithUncached {
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = {
|
||||
(a_type === acquireWriteUncached)
|
||||
}
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
class MESICoherence extends CoherencePolicyWithUncached {
|
||||
@ -554,9 +595,9 @@ class MESICoherence extends CoherencePolicyWithUncached {
|
||||
val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() }
|
||||
|
||||
val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() }
|
||||
val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() }
|
||||
val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() }
|
||||
val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() }
|
||||
val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() }
|
||||
val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
|
||||
@ -621,7 +662,9 @@ class MESICoherence extends CoherencePolicyWithUncached {
|
||||
def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id)
|
||||
def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask)
|
||||
def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op)
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
|
||||
val (read, write) = cpuCmdToRW(cmd)
|
||||
@ -678,6 +721,12 @@ class MESICoherence extends CoherencePolicyWithUncached {
|
||||
acquireAtomicUncached -> grantAtomicUncached
|
||||
))
|
||||
}
|
||||
def getGrantType(rel: Release, count: UFix): Bits = {
|
||||
MuxLookup(rel.r_type, grantReadUncached, Array(
|
||||
releaseVoluntaryInvalidateData -> grantVoluntaryAck
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = {
|
||||
MuxLookup(a_type, probeCopy, Array(
|
||||
@ -700,6 +749,9 @@ class MESICoherence extends CoherencePolicyWithUncached {
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = {
|
||||
(a_type === acquireWriteUncached)
|
||||
}
|
||||
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
class MigratoryCoherence extends CoherencePolicyWithUncached {
|
||||
@ -707,9 +759,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached {
|
||||
val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() }
|
||||
|
||||
val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() }
|
||||
val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() }
|
||||
val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() }
|
||||
val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() }
|
||||
val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() }
|
||||
val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() }
|
||||
val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached)
|
||||
|
||||
@ -789,7 +841,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached {
|
||||
def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id)
|
||||
def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask)
|
||||
def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op)
|
||||
def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id)
|
||||
def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached
|
||||
def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData
|
||||
|
||||
def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = {
|
||||
val (read, write) = cpuCmdToRW(cmd)
|
||||
@ -848,6 +902,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached {
|
||||
acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI?
|
||||
))
|
||||
}
|
||||
def getGrantType(rel: Release, count: UFix): Bits = {
|
||||
MuxLookup(rel.r_type, grantReadUncached, Array(
|
||||
releaseVoluntaryInvalidateData -> grantVoluntaryAck
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
def getProbeType(a_type: UFix, global_state: UFix): UFix = {
|
||||
MuxLookup(a_type, probeCopy, Array(
|
||||
@ -871,4 +931,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached {
|
||||
def needsAckReply(a_type: UFix, global_state: UFix): Bool = {
|
||||
(a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers)
|
||||
}
|
||||
def requiresAck(grant: Grant) = Bool(true)
|
||||
def requiresAck(release: Release) = Bool(false)
|
||||
}
|
||||
|
||||
@ -105,19 +105,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon
|
||||
override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] }
|
||||
}
|
||||
|
||||
abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data)
|
||||
class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data)
|
||||
class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()}
|
||||
|
||||
class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle {
|
||||
val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }}
|
||||
val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }}
|
||||
val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }}
|
||||
val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }}
|
||||
val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }}
|
||||
val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }}
|
||||
val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }}
|
||||
val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }}
|
||||
override def clone = { new TileLinkIO().asInstanceOf[this.type] }
|
||||
}
|
||||
}
|
||||
|
||||
@ -79,8 +79,20 @@ class Probe extends PhysicalAddress {
|
||||
val master_xact_id = Bits(width = MASTER_XACT_ID_BITS)
|
||||
}
|
||||
|
||||
class Release extends Bundle {
|
||||
object Release
|
||||
{
|
||||
def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = {
|
||||
val rel = new Release
|
||||
rel.r_type := r_type
|
||||
rel.addr := addr
|
||||
rel.client_xact_id := client_xact_id
|
||||
rel.master_xact_id := master_xact_id
|
||||
rel
|
||||
}
|
||||
}
|
||||
class Release extends PhysicalAddress {
|
||||
val r_type = Bits(width = RELEASE_TYPE_MAX_BITS)
|
||||
val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS)
|
||||
val master_xact_id = Bits(width = MASTER_XACT_ID_BITS)
|
||||
}
|
||||
|
||||
@ -90,7 +102,6 @@ class Grant extends MemData {
|
||||
val g_type = Bits(width = GRANT_TYPE_MAX_BITS)
|
||||
val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS)
|
||||
val master_xact_id = Bits(width = MASTER_XACT_ID_BITS)
|
||||
val require_ack = Bool()
|
||||
}
|
||||
|
||||
class GrantAck extends Bundle {
|
||||
|
||||
@ -288,7 +288,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id)
|
||||
grant.bits.payload.master_xact_id := UFix(0) // don't care
|
||||
grant.bits.payload.data := io.mem.resp.bits.data
|
||||
grant.bits.payload.require_ack := Bool(true)
|
||||
grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready
|
||||
|
||||
io.tiles(0).abort.valid := Bool(false)
|
||||
@ -368,7 +367,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co
|
||||
rep.bits.payload.client_xact_id := UFix(0)
|
||||
rep.bits.payload.master_xact_id := UFix(0)
|
||||
rep.bits.payload.data := io.mem.resp.bits.data
|
||||
rep.bits.payload.require_ack := Bool(true)
|
||||
rep.valid := Bool(false)
|
||||
when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) {
|
||||
rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx))
|
||||
@ -537,11 +535,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
{
|
||||
implicit val lnConf = conf.ln
|
||||
val co = conf.co
|
||||
val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_))
|
||||
val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_))
|
||||
val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY
|
||||
val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY
|
||||
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
val t = trackerList(i)
|
||||
t.io.tile_incoherent := io.incoherent.toBits
|
||||
t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i))
|
||||
@ -559,12 +557,12 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() }
|
||||
val abort_state = Reg(resetVal = s_idle)
|
||||
val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES)))
|
||||
val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_)
|
||||
val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_)
|
||||
val all_busy = trackerList.map(_.io.busy).reduce(_&&_)
|
||||
val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)))
|
||||
val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)))
|
||||
|
||||
val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() }
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() }
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
alloc_arb.io.in(i).valid := !trackerList(i).io.busy
|
||||
trackerList(i).io.acquire.bits := acquire.bits
|
||||
trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready
|
||||
@ -610,8 +608,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
}
|
||||
|
||||
// Handle probe request generation
|
||||
val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }}
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }}
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
val t = trackerList(i).io
|
||||
probe_arb.io.in(i).bits := t.probe.bits
|
||||
probe_arb.io.in(i).valid := t.probe.valid
|
||||
@ -622,13 +620,16 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
// Handle probe replies, which may or may not have data
|
||||
val release = io.network.release
|
||||
val release_data = io.network.release_data
|
||||
val idx = release.bits.payload.master_xact_id
|
||||
val voluntary = co.isVoluntary(release.bits.payload)
|
||||
val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_)
|
||||
val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b}
|
||||
val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id)
|
||||
release.ready := trackerList.map(_.io.release.ready).reduce(_||_)
|
||||
release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_)
|
||||
release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload)
|
||||
release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id
|
||||
release_data_dep_q.io.enq.bits.master_xact_id := idx
|
||||
release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_)
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
trackerList(i).io.release_data.valid := release_data.valid
|
||||
trackerList(i).io.release_data.bits := release_data.bits
|
||||
trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid
|
||||
@ -639,28 +640,27 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
|
||||
// Reply to initial requestor
|
||||
// Forward memory responses from mem to tile or arbitrate to ack
|
||||
val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }}
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }}
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
val t = trackerList(i).io
|
||||
grant_arb.io.in(i).bits := t.grant.bits
|
||||
grant_arb.io.in(i).valid := t.grant.valid
|
||||
t.grant.ready := grant_arb.io.in(i).ready
|
||||
}
|
||||
grant_arb.io.out.ready := Bool(false)
|
||||
io.network.grant.valid := grant_arb.io.out.valid
|
||||
io.network.grant.bits := grant_arb.io.out.bits
|
||||
grant_arb.io.out.ready := io.network.grant.ready
|
||||
when(io.mem.resp.valid) {
|
||||
io.network.grant.valid := Bool(true)
|
||||
io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag)
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready
|
||||
}
|
||||
}
|
||||
|
||||
// Free finished transactions
|
||||
val ack = io.network.grant_ack
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i))
|
||||
}
|
||||
ack.ready := Bool(true)
|
||||
@ -668,9 +668,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
// Create an arbiter for the one memory port
|
||||
// We have to arbitrate between the different trackers' memory requests
|
||||
// and once we have picked a request, get the right write data
|
||||
val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() }
|
||||
val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() }
|
||||
for( i <- 0 until NGLOBAL_XACTS ) {
|
||||
val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() }
|
||||
val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() }
|
||||
for( i <- 0 to NGLOBAL_XACTS ) {
|
||||
mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd
|
||||
mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data
|
||||
mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock
|
||||
@ -679,7 +679,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren
|
||||
io.mem.req_data <> Queue(mem_req_data_arb.io.out)
|
||||
}
|
||||
|
||||
class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component {
|
||||
|
||||
abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator {
|
||||
val co = conf.co
|
||||
implicit val ln = conf.ln
|
||||
val io = new Bundle {
|
||||
@ -699,15 +700,83 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }}
|
||||
val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }}
|
||||
val busy = Bool(OUTPUT)
|
||||
val has_conflict = Bool(OUTPUT)
|
||||
val has_acquire_conflict = Bool(OUTPUT)
|
||||
val has_release_conflict = Bool(OUTPUT)
|
||||
}
|
||||
}
|
||||
|
||||
class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) {
|
||||
val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() }
|
||||
val state = Reg(resetVal = s_idle)
|
||||
val xact = Reg{ new Release }
|
||||
val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles)))
|
||||
val release_data_needs_write = Reg(resetVal = Bool(false))
|
||||
val mem_cmd_sent = Reg(resetVal = Bool(false))
|
||||
|
||||
io.acquire.ready := Bool(false)
|
||||
io.acquire_data.ready := Bool(false)
|
||||
io.acquire_data_dep.ready := Bool(false)
|
||||
io.mem_resp.ready := Bool(false)
|
||||
io.probe.valid := Bool(false)
|
||||
io.busy := Bool(true)
|
||||
io.has_acquire_conflict := Bool(false)
|
||||
io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle)
|
||||
|
||||
io.mem_req_cmd.valid := Bool(false)
|
||||
io.mem_req_cmd.bits.rw := Bool(false)
|
||||
io.mem_req_cmd.bits.addr := xact.addr
|
||||
io.mem_req_cmd.bits.tag := UFix(id)
|
||||
io.mem_req_data.valid := Bool(false)
|
||||
io.mem_req_data.bits.data := UFix(0)
|
||||
io.mem_req_lock := Bool(false)
|
||||
io.release.ready := Bool(false)
|
||||
io.release_data.ready := Bool(false)
|
||||
io.release_data_dep.ready := Bool(false)
|
||||
io.grant.valid := Bool(false)
|
||||
io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0))
|
||||
io.grant.bits.payload.client_xact_id := xact.client_xact_id
|
||||
io.grant.bits.payload.master_xact_id := UFix(id)
|
||||
io.grant.bits.header.dst := init_client_id_
|
||||
|
||||
switch (state) {
|
||||
is(s_idle) {
|
||||
when( io.release.valid ) {
|
||||
xact := io.release.bits.payload
|
||||
init_client_id_ := io.release.bits.header.src
|
||||
release_data_needs_write := co.messageHasData(io.release.bits.payload)
|
||||
mem_cnt := UFix(0)
|
||||
mem_cmd_sent := Bool(false)
|
||||
io.release.ready := Bool(true)
|
||||
state := s_mem
|
||||
}
|
||||
}
|
||||
is(s_mem) {
|
||||
when (release_data_needs_write) {
|
||||
doMemReqWrite(io.mem_req_cmd,
|
||||
io.mem_req_data,
|
||||
io.mem_req_lock,
|
||||
io.release_data,
|
||||
release_data_needs_write,
|
||||
mem_cmd_sent,
|
||||
io.release_data_dep.ready,
|
||||
io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)))
|
||||
} . otherwise { state := s_ack }
|
||||
}
|
||||
is(s_ack) {
|
||||
io.grant.valid := Bool(true)
|
||||
when(io.grant.ready) { state := s_idle }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) {
|
||||
val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() }
|
||||
val state = Reg(resetVal = s_idle)
|
||||
val xact = Reg{ new Acquire }
|
||||
val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles)))
|
||||
//TODO: Will need id reg for merged release xacts
|
||||
val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles)))
|
||||
val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_)
|
||||
val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles)))
|
||||
val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles))
|
||||
val x_needs_read = Reg(resetVal = Bool(false))
|
||||
@ -715,9 +784,6 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
val release_data_needs_write = Reg(resetVal = Bool(false))
|
||||
val x_w_mem_cmd_sent = Reg(resetVal = Bool(false))
|
||||
val p_w_mem_cmd_sent = Reg(resetVal = Bool(false))
|
||||
val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES)))
|
||||
val mem_cnt_next = mem_cnt + UFix(1)
|
||||
val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES))
|
||||
val probe_initial_flags = Bits(width = conf.ln.nTiles)
|
||||
probe_initial_flags := Bits(0)
|
||||
if (conf.ln.nTiles > 1) {
|
||||
@ -730,10 +796,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0)))
|
||||
probe_initial_flags := ~(io.tile_incoherent | myflag)
|
||||
}
|
||||
val all_grants_require_acks = Bool(true)
|
||||
|
||||
io.busy := state != s_idle
|
||||
io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle)
|
||||
io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle)
|
||||
io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle)
|
||||
io.mem_req_cmd.valid := Bool(false)
|
||||
io.mem_req_cmd.bits.rw := Bool(false)
|
||||
io.mem_req_cmd.bits.addr := xact.addr
|
||||
@ -747,10 +813,9 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
io.probe.bits.payload.addr := xact.addr
|
||||
io.probe.bits.header.dst := UFix(0)
|
||||
io.grant.bits.payload.data := io.mem_resp.bits.data
|
||||
io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_)
|
||||
io.grant.bits.payload.g_type := grant_type
|
||||
io.grant.bits.payload.client_xact_id := xact.client_xact_id
|
||||
io.grant.bits.payload.master_xact_id := UFix(id)
|
||||
io.grant.bits.payload.require_ack := all_grants_require_acks
|
||||
io.grant.bits.header.dst := init_client_id_
|
||||
io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag))
|
||||
io.acquire.ready := Bool(false)
|
||||
@ -821,12 +886,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
doMemReqRead(io.mem_req_cmd, x_needs_read)
|
||||
} . otherwise {
|
||||
state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack,
|
||||
Mux(all_grants_require_acks, s_busy, s_idle))
|
||||
Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle))
|
||||
}
|
||||
}
|
||||
is(s_ack) {
|
||||
io.grant.valid := Bool(true)
|
||||
when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) }
|
||||
when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) }
|
||||
}
|
||||
is(s_busy) { // Nothing left to do but wait for transaction to complete
|
||||
when (io.free) {
|
||||
@ -834,6 +899,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait MemoryRequestGenerator {
|
||||
val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES)))
|
||||
val mem_cnt_next = mem_cnt + UFix(1)
|
||||
val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES))
|
||||
|
||||
def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) {
|
||||
req_cmd.bits.rw := Bool(true)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user