From 71c8d3fd4100ba59f6d0eb7f0377eb83bc26dd4a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Feb 2012 15:13:08 -0800 Subject: [PATCH 001/374] reorganize directory structure From 76cacbd9918397c85c7fce56571a8658dd9c827c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 14 Feb 2012 15:51:32 -0800 Subject: [PATCH 002/374] Added coherence tile function defs, with traits and constants --- uncore/coherence.scala | 128 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 uncore/coherence.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala new file mode 100644 index 00000000..460214e1 --- /dev/null +++ b/uncore/coherence.scala @@ -0,0 +1,128 @@ +package Top { + +import Chisel._ +import Constants._ + +class TransactionInit extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionAbort extends Bundle { + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends Bundle { + val ptype = Bits(width = 2) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) +} + +class ProbeReply extends Bundle { + val ptype = Bits(width = 2) + val hasData = Bool() + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionReply extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionFinish extends Bundle { + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = new TransactionInit().asOutput + val xact_abort = new TransactionAbort().asInput + val probe_req = new ProbeRequest().asInput + val probe_rep = new ProbeReply().asOutput + val xact_rep = new TransactionReply().asInput + val xact_finish = new TransactionFinish().asOutput +} + +trait ThreeStateIncoherence { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + + def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { + val store = (cmd === M_XWR) + val load = (cmd === M_XRD) + val amo = cmd(3).toBool + val read = load || amo || (cmd === M_PFR) + val write = store || amo || (cmd === M_PFW) + (read, write) + } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + ( state === tileClean || state === tileDirty) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsWriteback (state: UFix): Bool = { + state === tileDirty + } + + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newState(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + } + def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, state) + } + +} + +trait FourStateCoherence { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val is_hit = Bool(false) + switch(cmd) { + is(M_XRD) { + is_hit := state === tileShared || + state === tileExclusiveClean || + state === tileExclusiveDirty + } + is(M_XWR) { + is_hit := state === tileExclusiveClean || + state === tileExclusiveDirty + } + } + is_hit + } + + def needsWriteback (state: UFix): Bool = { + state === tileExclusiveDirty + } + + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool + + def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { + val state = UFix(0) + switch(incoming.ptype) { + is(probeInvalidate) { state := tileInvalid } + is(probeDowngrade) { state := tileShared } + } + state.toBits + } +} + +} From ecbf47fe16327ae8a6c4453556a42b989df59dea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Feb 2012 13:54:36 -0800 Subject: [PATCH 003/374] Replace nbcache manipulation of meta state bits with abstracted functions --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 460214e1..7419555e 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -76,7 +76,7 @@ trait ThreeStateIncoherence { def newStateOnFlush() = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) From cfb818aa97e0c96a0b8bb033a5680d442b344433 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 16 Feb 2012 12:59:38 -0800 Subject: [PATCH 004/374] Abstract class for coherence policies --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7419555e..b55ac9a1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -47,9 +47,7 @@ class ioTileLink extends Bundle { val xact_finish = new TransactionFinish().asOutput } -trait ThreeStateIncoherence { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - +trait CoherencePolicy { def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) @@ -58,6 +56,10 @@ trait ThreeStateIncoherence { val write = store || amo || (cmd === M_PFW) (read, write) } +} + +trait ThreeStateIncoherence extends CoherencePolicy { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -87,32 +89,35 @@ trait ThreeStateIncoherence { } -trait FourStateCoherence { +trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { - val is_hit = Bool(false) - switch(cmd) { - is(M_XRD) { - is_hit := state === tileShared || - state === tileExclusiveClean || - state === tileExclusiveDirty - } - is(M_XWR) { - is_hit := state === tileExclusiveClean || - state === tileExclusiveDirty - } - } - is_hit + val (read, write) = cpuCmdToRW(cmd) + ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || + (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid } def needsWriteback (state: UFix): Bool = { state === tileExclusiveDirty } + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + + // TODO: New funcs as compared to incoherent protocol: + def newState(cmd: Bits, state: UFix): UFix + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnPrimaryMiss(cmd: Bits): UFix + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { From b9c42a80c87dfae75110004a651d3bccc3217bc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 10:12:13 -0800 Subject: [PATCH 005/374] Added coherence message type enums --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b55ac9a1..9fcf32eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) val data = Bits(width = MEM_DATA_BITS) @@ -15,20 +15,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) } class TransactionReply extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) From 33a26424bd1cdc52298a97dcc30952b8fe30d05a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 12:14:57 -0800 Subject: [PATCH 006/374] Refining tilelink interface --- uncore/coherence.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fcf32eb..cfde62ba 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -39,12 +39,12 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = new TransactionInit().asOutput - val xact_abort = new TransactionAbort().asInput - val probe_req = new ProbeRequest().asInput - val probe_rep = new ProbeReply().asOutput - val xact_rep = new TransactionReply().asInput - val xact_finish = new TransactionFinish().asOutput + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,4 +130,6 @@ trait FourStateCoherence extends CoherencePolicy { } } + + } From c3646e2e6411c3003a9683b4674a023b780c3848 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 18:24:52 -0800 Subject: [PATCH 007/374] Improved TileIO organization, beginnings of hub implementation --- uncore/coherence.scala | 92 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index cfde62ba..776a64e0 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -24,6 +24,9 @@ class ProbeReply extends Bundle { val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -31,6 +34,9 @@ class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class TransactionReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -39,12 +45,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,6 +138,78 @@ trait FourStateCoherence extends CoherencePolicy { } } +class XactTracker(id: Int) extends Component { + val io = new Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit() } + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip + val mem_req = (new ioDecoupled) { new MemReq() }.flip + val xact_finish = Bool(INPUT) + val tile_id_in = Bits(TILE_ID_BITS, INPUT) + val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) + val ongoing_addr = Bits(PADDR_BITS, OUTPUT) + val busy = Bool(OUTPUT) + } + + val valid = Reg(resetVal = Bool(false)) + val addr = Reg{ Bits() } + val ttype = Reg{ Bits() } + val tile_id = Reg{ Bits() } + val tile_xact_id = Reg{ Bits() } + val probe_done = Reg{ Bits() } + +} + +abstract class CoherenceHub extends Component + +class CoherenceHubNoDir extends CoherenceHub { + val io = new Bundle { + val tiles = Vec(NTILES) { new ioTileLink() } + val mem = new ioDCache().flip + } + + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + + // In parallel, every cycle: nack conflicting transactions, free finished ones + for( j <- 0 until NTILES ) { + val init = io.tiles(j).xact_init + val abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + val busys = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busys(i) := t.busy + conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + } + abort.valid := conflicts.orR || busys.andR + abort.bits.tileTransactionID := init.bits.tileTransactionID + //if abort.rdy, init.pop() + + } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + val freed = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + } + t.xact_finish := freed.orR + //finish.pop() + } + + // Forward memory responses from mem to tile + //for( j <- until NTILES ) { + // tiles(j).xact_rep.ttype = + // tiles(j).xact_rep.tileTransactionID = + // tiles(j).xact_rep.globalTransactionID = + // val data = Bits + // + // Pick a single request of these types to process + //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } + //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } } + +} From ffb88467db080f48cb2d62f7a308613b50440cde Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 17:49:28 -0800 Subject: [PATCH 008/374] finished xact_rep transactor in coherence hub --- uncore/coherence.scala | 96 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 776a64e0..3744fdb6 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,6 +2,7 @@ package Top { import Chisel._ import Constants._ +import hwacha.GenArray class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -146,10 +147,12 @@ class XactTracker(id: Int) extends Component { val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip val mem_req = (new ioDecoupled) { new MemReq() }.flip val xact_finish = Bool(INPUT) - val tile_id_in = Bits(TILE_ID_BITS, INPUT) - val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) - val ongoing_addr = Bits(PADDR_BITS, OUTPUT) val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val ttype = Bits(TTYPE_BITS, OUTPUT) } val valid = Reg(resetVal = Bool(false)) @@ -164,47 +167,104 @@ class XactTracker(id: Int) extends Component { abstract class CoherenceHub extends Component class CoherenceHubNoDir extends CoherenceHub { + + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { + addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) + } + def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + val ret = Wire() { Bits(width = TTYPE_BITS) } + switch (ttype) { + is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } + is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } + is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } + is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } + } + ret + } + val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + for( i <- 0 until NGLOBAL_XACTS) { + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) + ttype_arr.write( UFix(i), trackerList(i).io.ttype) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + } // In parallel, every cycle: nack conflicting transactions, free finished ones for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) - val busys = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - busys(i) := t.busy - conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := conflicts.orR || busys.andR + abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) abort.bits.tileTransactionID := init.bits.tileTransactionID - //if abort.rdy, init.pop() - + // TODO: + // Reg(aborted) := (abort.ready && abort.valid) + // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) + // init.rdy = aborted || allocated } + +/* +// Todo: which implementation is clearer? for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io val freed = Bits(width = NTILES) for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + finish.ready := Bool(true) // finsh.pop() } t.xact_finish := freed.orR - //finish.pop() + } +*/ + + free_arr := Bits(0, width=NGLOBAL_XACTS) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + when(finish.valid) { + free_arr.write(finish.bits.globalTransactionID, Bool(true)) + } + finish.ready := Bool(true) } // Forward memory responses from mem to tile - //for( j <- until NTILES ) { - // tiles(j).xact_rep.ttype = - // tiles(j).xact_rep.tileTransactionID = - // tiles(j).xact_rep.globalTransactionID = - // val data = Bits - // + val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val xrep_cnt_next = xrep_cnt + UFix(1) + when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } + val idx = io.mem.resp_tag + val readys = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data + readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) + val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val + io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) + io.tiles(j).xact_rep_data.valid := this_rep_valid + } + // If there were a ready signal due to e.g. intervening network: + //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } From 1dcf25586f9997f512cf88f025174262b12f9c52 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 18:12:50 -0800 Subject: [PATCH 009/374] finished xact_finish and xact_abort transactors in coherence hub --- uncore/coherence.scala | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3744fdb6..534c4bb9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -205,7 +205,9 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.xact_finish := free_arr.read(UFix(i)) } - // In parallel, every cycle: nack conflicting transactions, free finished ones + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + val initiating = Wire() { Bits(width = NTILES) } for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort @@ -214,34 +216,16 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + abort.valid := init.valid && aborting abort.bits.tileTransactionID := init.bits.tileTransactionID - // TODO: - // Reg(aborted) := (abort.ready && abort.valid) - // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) - // init.rdy = aborted || allocated + init.ready := aborting(j) || initiating(j) } - -/* -// Todo: which implementation is clearer? - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - val freed = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - val finish = io.tiles(j).xact_finish - free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) - finish.ready := Bool(true) // finsh.pop() - } - t.xact_finish := freed.orR - } -*/ - free_arr := Bits(0, width=NGLOBAL_XACTS) + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - when(finish.valid) { - free_arr.write(finish.bits.globalTransactionID, Bool(true)) - } + free_arr.write(finish.bits.globalTransactionID, finish.valid) finish.ready := Bool(true) } From b3cf8f3f354b064c139ae040038d70c438542d19 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 12:56:09 -0800 Subject: [PATCH 010/374] Better abstraction of data bundles --- uncore/coherence.scala | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 534c4bb9..579af518 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,13 +4,27 @@ import Chisel._ import Constants._ import hwacha.GenArray +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) - val data = Bits(width = MEM_DATA_BITS) } +class TransactionInitData extends MemData + class TransactionAbort extends Bundle { val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) } @@ -27,9 +41,7 @@ class ProbeReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class ProbeReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class ProbeReplyData extends MemData class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -37,9 +49,7 @@ class TransactionReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class TransactionReplyData extends MemData class TransactionFinish extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) @@ -47,6 +57,7 @@ class TransactionFinish extends Bundle { class ioTileLink extends Bundle { val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val xact_abort = (new ioDecoupled) { new TransactionAbort() } val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip From dfdcb7c696df600ddc9a234bf189fbd2600b3202 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:09 -0800 Subject: [PATCH 011/374] Better foldR --- uncore/coherence.scala | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 579af518..bb2f7d65 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -18,41 +18,43 @@ class MemData extends Bundle { } class TransactionInit extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class TransactionInitData extends MemData class TransactionAbort extends Bundle { - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } class ProbeRequest extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val hasData = Bool() - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val has_data = Bool() + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class TransactionReplyData extends MemData class TransactionFinish extends Bundle { - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ioTileLink extends Bundle { @@ -142,7 +144,7 @@ trait FourStateCoherence extends CoherencePolicy { def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { val state = UFix(0) - switch(incoming.ptype) { + switch(incoming.p_type) { is(probeInvalidate) { state := tileInvalid } is(probeDowngrade) { state := tileShared } } From 8856e2b8bb7b5c5851d227704e8182c50ee1f01a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:53 -0800 Subject: [PATCH 012/374] More stylish bundle param names, some hub progress --- uncore/coherence.scala | 105 ++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 27 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bb2f7d65..d7597cdf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,23 +154,27 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip - val mem_req = (new ioDecoupled) { new MemReq() }.flip + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val x_init_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val x_init_data_idx = Bits(log2up(NTILES), INPUT) + val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val ttype = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val pop_p_rep = Bool(OUTPUT) + val pop_p_rep_data = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } - val ttype = Reg{ Bits() } + val t_type = Reg{ Bits() } val tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } @@ -184,9 +188,9 @@ class CoherenceHubNoDir extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } - def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (ttype) { + switch (t_type) { is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } @@ -205,17 +209,27 @@ class CoherenceHubNoDir extends CoherenceHub { val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) - tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) - ttype_arr.write( UFix(i), trackerList(i).io.ttype) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + t_type_arr.write( UFix(i), trackerList(i).io.t_type) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) + trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) + trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } // Nack conflicting transaction init attempts @@ -231,14 +245,14 @@ class CoherenceHubNoDir extends CoherenceHub { } aborting(j) := (conflicts.orR || busy_arr.flatten().andR) abort.valid := init.valid && aborting - abort.bits.tileTransactionID := init.bits.tileTransactionID + abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - free_arr.write(finish.bits.globalTransactionID, finish.valid) + do_free_arr.write(finish.bits.global_xact_id, finish.valid) finish.ready := Bool(true) } @@ -249,18 +263,55 @@ class CoherenceHubNoDir extends CoherenceHub { val idx = io.mem.resp_tag val readys = Bits(width = NTILES) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val - io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) - io.tiles(j).xact_rep_data.valid := this_rep_valid + io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) } - // If there were a ready signal due to e.g. intervening network: + // If there were a ready signal due to e.g. intervening network use: //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + + val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + } + mem_req_arb.io.out.ready := io.mem.req_rdy + io.mem.req_val := mem_req_arb.io.out.valid + io.mem.req_rw := mem_req_arb.io.out.bits.rw + io.mem.req_tag := mem_req_arb.io.out.bits.tag + io.mem.req_addr := mem_req_arb.io.out.bits.addr + io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + Bits(0, width = MEM_DATA_BITS), + (0 until NTILES).map( j => + UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + io.tiles(j).probe_rep_data.bits.data, + io.tiles(j).xact_init_data.bits.data))) + + for( j <- 0 until NTILES ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_data_idx_arr.write(idx, UFix(j)) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES) { + val p_rep = io.tiles(j).probe_rep + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + rep_cnt_dec_arr.write(UFix(i), flags) + } + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } From ca2e70454e7ad132e02478dac45cbfbb426c1f4d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 25 Feb 2012 17:09:26 -0800 Subject: [PATCH 013/374] change package name and sbt project name to rocket --- uncore/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7597cdf..d419dd52 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -1,4 +1,4 @@ -package Top { +package rocket import Chisel._ import Constants._ @@ -319,5 +319,3 @@ class CoherenceHubNoDir extends CoherenceHub { } - -} From e22106af3f3dfc2049d4214a93e60ee5fbd3616a Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:08 -0800 Subject: [PATCH 014/374] updating rocket code to lastest version of chisel, passes assembly tests in C++ and Verilog as long as you dont use the vector unit --- uncore/coherence.scala | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d419dd52..af0d27c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,7 +2,6 @@ package rocket import Chisel._ import Constants._ -import hwacha.GenArray class HubMemReq extends Bundle { val rw = Bool() @@ -205,18 +204,18 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) @@ -243,7 +242,7 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.toBits().andR) abort.valid := init.valid && aborting abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) From 0fd777f4807f1bf67d69fcbcac2a03aa14124da4 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:23 -0800 Subject: [PATCH 015/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From b6e6d603cc77e6ab0cd46237922709b61b6ba562 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:34:40 -0800 Subject: [PATCH 016/374] xact init transactors in coherence hub --- uncore/coherence.scala | 125 +++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index af0d27c8..520f7c05 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -12,6 +12,12 @@ class HubMemReq extends Bundle { val is_probe_rep = Bool() } +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -154,6 +160,8 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) val x_init_has_data = Bool(INPUT) @@ -162,22 +170,27 @@ class XactTracker(id: Int) extends Component { val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) - val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) - val pop_p_rep = Bool(OUTPUT) - val pop_p_rep_data = Bool(OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) + val pop_x_init_data = Bool(OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } val t_type = Reg{ Bits() } - val tile_id = Reg{ Bits() } + val init_tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } + //TODO: Decrement the probe count when final data piece is written + // Connent io.mem.ready sig to correct pop* outputs + // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component @@ -204,23 +217,23 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) @@ -231,23 +244,6 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } - // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } - val initiating = Wire() { Bits(width = NTILES) } - for( j <- 0 until NTILES ) { - val init = io.tiles(j).xact_init - val abort = io.tiles(j).xact_abort - val conflicts = Bits(width = NGLOBAL_XACTS) - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) - } - aborting(j) := (conflicts.orR || busy_arr.toBits().andR) - abort.valid := init.valid && aborting - abort.bits.tile_xact_id := init.bits.tile_xact_id - init.ready := aborting(j) || initiating(j) - } - // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -255,6 +251,7 @@ class CoherenceHubNoDir extends CoherenceHub { finish.ready := Bool(true) } + // Reply to initial requestor // Forward memory responses from mem to tile val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val xrep_cnt_next = xrep_cnt + UFix(1) @@ -267,11 +264,11 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -293,28 +290,70 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).probe_rep_data.bits.data, io.tiles(j).xact_init_data.bits.data))) + // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) p_rep_data_idx_arr.write(idx, UFix(j)) - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } rep_cnt_dec_arr.write(UFix(i), flags) } + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + // Don't abort writebacks stalled on mem. + // TODO: This assumes overlapped writeback init reqs to + // the same addr will never be issued; is this ok? + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + val want_to_abort = conflicts.orR || busy_arr.flatten().andR + x_abort.valid := want_to_abort && x_init.valid + aborting(j) := want_to_abort && x_abort.ready + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + } - // Pick a single request of these types to process - //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } - //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.init_tile_id := UFix(j) + init_arb.io.in(j).bits.data_valid := x_init_data.valid + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) } From 70cdf869ac94af7c50405218cb0cf1a87cc76f0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:53:07 -0800 Subject: [PATCH 017/374] probe req transactors in coherence hub --- uncore/coherence.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 520f7c05..8cc616de 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -161,6 +161,7 @@ class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) @@ -174,6 +175,7 @@ class XactTracker(id: Int) extends Component { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) @@ -356,4 +358,17 @@ class CoherenceHubNoDir extends CoherenceHub { !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + // Handle probe request generation + // Must arbitrate for each request port + for( j <- 0 until NTILES ) { + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.probe_req.bits + p_req_arb.io.in(i).ready := t.probe_req.ready + p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + } + p_req_arb.io.out <> io.tiles(j).probe_req + } + } From 3393dc2d31aaa857daf3d604c60e8d95e5493450 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 11:26:18 -0800 Subject: [PATCH 018/374] Added probe_req ready sigs, GenArray to Vec --- uncore/coherence.scala | 68 ++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8cc616de..8aff1592 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -168,7 +168,8 @@ class XactTracker(id: Int) extends Component { val x_init_has_data = Bool(INPUT) val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val rep_cnt_dec = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -218,32 +219,35 @@ class CoherenceHubNoDir extends CoherenceHub { val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) - trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } // Free finished transactions @@ -308,7 +312,7 @@ class CoherenceHubNoDir extends CoherenceHub { val p_rep = io.tiles(j).probe_rep flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } - rep_cnt_dec_arr.write(UFix(i), flags) + p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -326,7 +330,7 @@ class CoherenceHubNoDir extends CoherenceHub { // the same addr will never be issued; is this ok? } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.flatten().andR + val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid aborting(j) := want_to_abort && x_abort.ready } @@ -354,21 +358,27 @@ class CoherenceHubNoDir extends CoherenceHub { x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) // Handle probe request generation // Must arbitrate for each request port + val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) for( j <- 0 until NTILES ) { - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.probe_req.bits - p_req_arb.io.in(i).ready := t.probe_req.ready - p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) } - p_req_arb.io.out <> io.tiles(j).probe_req + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + flags(j) := p_req_arb_arr(j).io.in(i).ready + } + p_rep_cnt_dec_arr.write(UFix(i), flags) } } From 7d331858e25c840e19fb745b01b0465137230901 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 27 Feb 2012 18:36:09 -0800 Subject: [PATCH 019/374] replace ioDCache with ioMem --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8aff1592..d5a5e414 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -216,7 +216,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioDCache().flip + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) From ffc7652e0166b38b4db2612e93358dd267e22206 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 19:10:15 -0800 Subject: [PATCH 020/374] Null coherence hub. Begin work on internal tracker logic --- uncore/coherence.scala | 163 +++++++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 30 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d5a5e414..20c95e78 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -157,46 +157,149 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component { +class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) - val x_init_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } - val valid = Reg(resetVal = Bool(false)) - val addr = Reg{ Bits() } - val t_type = Reg{ Bits() } - val init_tile_id = Reg{ Bits() } - val tile_xact_id = Reg{ Bits() } - val probe_done = Reg{ Bits() } + val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ Bits() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val probe_done = Reg{ Bits() } + val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.t_type := t_type_ + +/* +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +*/ +/* + when( alloc_req.valid && can_alloc ) { + valid := Bool(true) + addr := alloc_req.bits.xact_init.address + t_type := alloc_req.bits.xact_init.t_type + init_tile_id := alloc_req.bits.init_tile_id + tile_xact_id := alloc_req.bits.xact_init.tile_xact_id + [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 + } + when ( alloc_req.bits.data_valid ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + } + when( p_rep_has_data ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + + } + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + push_p_req = Bits(0, width = NTILES) + pop_p_rep = Bits(0, width = NTILES) + pop_p_rep_data = Bits(0, width = NTILES) + pop_x_init = Bool(false) + pop_x_init_data = Bool(false) + send_x_rep_ack = Bool(false) + + + } +*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component +abstract class CoherenceHub extends Component with CoherencePolicy + +class CoherenceHubNull extends Component { + val io = new Bundle { + val tile = new ioTileLink() + val mem = new ioMem() + } + + val x_init = io.tile.xact_init + val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + x_init.ready := io.mem.req_rdy + io.mem.req_val := x_init.valid + io.mem.req_rw := is_write + io.mem.req_tag := x_init.bits.tile_xact_id + io.mem.req_addr := x_init.bits.address + + val x_rep = io.tile.xact_rep + x_rep.bits.t_type := Bits(width = TTYPE_BITS) + x_rep.bits.has_data := !is_write + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.valid := io.mem.resp_val + + //TODO: + val x_init_data = io.tile.xact_init_data + val x_rep_data = io.tile.xact_rep_data + x_init_data.ready := io.mem.req_rdy + io.mem.req_wdata := x_init_data.bits.data + x_rep_data.bits.data := io.mem.resp_data + x_rep_data.valid := io.mem.resp_val + // Should be: + //io.mem.req_data <> x_init_data + //x_rep_data <> io.mem.resp_data + +} + class CoherenceHubNoDir extends CoherenceHub { From e15284d22ce5dcef9a2a1af46e4de9c81d0a3566 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 28 Feb 2012 17:33:06 -0800 Subject: [PATCH 021/374] Added temporary ioMemHub and made coherence hub implementations depend on it rather than ioMem --- uncore/coherence.scala | 281 +++++++++++++++++++++++------------------ 1 file changed, 157 insertions(+), 124 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 20c95e78..11c764ec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,13 +3,39 @@ package rocket import Chisel._ import Constants._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(PADDR_BITS - OFFSET_BITS) + val tag = Bits(MEM_TAG_BITS) +} + +class MemResp () extends Bundle +{ + val tag = Bits(MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) + val valid = Bool() +} + +class ioMemHub() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val req_data = (new ioDecoupled) { new MemData() }.flip + val resp = new MemResp() +} + class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } +} + +class HubProbeRep extends Bundle { + val reply = (new ioDecoupled) { new ProbeReply } + val data_idx = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -18,9 +44,6 @@ class TrackerAllocReq extends Bundle { val data_valid = Bool() } -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) @@ -49,15 +72,13 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData -class TransactionReply extends Bundle { +class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends MemData - class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -70,7 +91,6 @@ class ioTileLink extends Bundle { val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -160,18 +180,21 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_rep = (new ioDecoupled) { new HubProbeRep() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val x_init_data = (new ioDecoupled) { new TransactionInitData() } val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(log2up(NTILES), INPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) @@ -183,8 +206,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val send_x_rep_ack = Bool(OUTPUT) } - - val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, + (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + } + + val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -192,73 +221,99 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val tile_xact_id_ = Reg{ Bits() } val probe_done = Reg{ Bits() } val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_rep_data_idx_ = Reg{ Bits() } + val x_init_data_needs_wb = Reg{ Bool() } + val p_rep_data_needs_wb = Reg{ Bool() } + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ -/* -class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() -*/ -/* - when( alloc_req.valid && can_alloc ) { - valid := Bool(true) - addr := alloc_req.bits.xact_init.address - t_type := alloc_req.bits.xact_init.t_type - init_tile_id := alloc_req.bits.init_tile_id - tile_xact_id := alloc_req.bits.xact_init.tile_xact_id - [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 - } - when ( alloc_req.bits.data_valid ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready - } - when( p_rep_has_data ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready + io.mem_req.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.bits.addr := addr_ + io.mem_req.bits.req_cmd.bits.tag := UFix(id) + // := io.mem.ready //sent mem req + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr + io.push_p_req := Bits(0, width = NTILES) + io.pop_p_rep := Bits(0, width = NTILES) + io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_x_init := Bool(false) + io.pop_x_init_data := Bool(false) + io.send_x_rep_ack := Bool(false) + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.init_tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + p_rep_count := UFix(NTILES) + p_req_flags := ~Bits(0, width = NTILES) + state := s_probe + io.pop_x_init := Bool(true) + } + } + is(s_mem_r) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_w) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_wr) { + when(io.probe_rep.bits.reply.bits.has_data) { + //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy + //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO + } . otherwise { + //io.pop_x_init := io.mem_req_rdy + //io.pop_x_init_data := io.mem_req_rdy + } + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_mem_r } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + when(io.p_rep_cnt_dec.orR) { + val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + p_rep_count := p_rep_count_next + when(p_rep_count_next === UFix(0)) { + state := s_busy //TODO: XXXXXXXXXX + } + } + when(p_rep_has_data) { + p_rep_data_needs_wb := Bool(true) + p_rep_data_idx_ := p_rep_data_idx + } + } + is(s_busy) { + when (io.xact_finish) { + state := s_idle + } + } } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - push_p_req = Bits(0, width = NTILES) - pop_p_rep = Bits(0, width = NTILES) - pop_p_rep_data = Bits(0, width = NTILES) - pop_x_init = Bool(false) - pop_x_init_data = Bool(false) - send_x_rep_ack = Bool(false) - - - } -*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt @@ -269,35 +324,24 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { val tile = new ioTileLink() - val mem = new ioMem() + val mem = new ioMemHub() } val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_rdy - io.mem.req_val := x_init.valid - io.mem.req_rw := is_write - io.mem.req_tag := x_init.bits.tile_xact_id - io.mem.req_addr := x_init.bits.address + x_init.ready := io.mem.req_cmd.ready + io.mem.req_cmd.valid := x_init.valid + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Bits(width = TTYPE_BITS) - x_rep.bits.has_data := !is_write - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.valid := io.mem.resp_val - - //TODO: - val x_init_data = io.tile.xact_init_data - val x_rep_data = io.tile.xact_rep_data - x_init_data.ready := io.mem.req_rdy - io.mem.req_wdata := x_init_data.bits.data - x_rep_data.bits.data := io.mem.resp_data - x_rep_data.valid := io.mem.resp_val - // Should be: - //io.mem.req_data <> x_init_data - //x_rep_data <> io.mem.resp_data - + x_rep.bits.data := io.mem.resp.data + x_rep.valid := io.mem.resp.valid } @@ -319,7 +363,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem + val mem = new ioMemHub } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -333,8 +377,6 @@ class CoherenceHubNoDir extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } @@ -347,8 +389,6 @@ class CoherenceHubNoDir extends CoherenceHub { sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) - trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } @@ -362,22 +402,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val xrep_cnt_next = xrep_cnt + UFix(1) - when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } - val idx = io.mem.resp_tag - val readys = Bits(width = NTILES) + val idx = io.mem.resp.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data - readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) + io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -387,33 +421,30 @@ class CoherenceHubNoDir extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { mem_req_arb.io.in(i) <> trackerList(i).io.mem_req } - mem_req_arb.io.out.ready := io.mem.req_rdy - io.mem.req_val := mem_req_arb.io.out.valid - io.mem.req_rw := mem_req_arb.io.out.bits.rw - io.mem.req_tag := mem_req_arb.io.out.bits.tag - io.mem.req_addr := mem_req_arb.io.out.bits.addr - io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - Bits(0, width = MEM_DATA_BITS), - (0 until NTILES).map( j => - UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - io.tiles(j).probe_rep_data.bits.data, - io.tiles(j).xact_init_data.bits.data))) + //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready + io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd + io.mem.req_data <> mem_req_arb.io.out.bits.req_data + //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + // Bits(0, width = MEM_DATA_BITS), + // (0 until NTILES).map( j => + // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + // io.tiles(j).probe_rep_data.bits.data, + // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) - p_rep_data_idx_arr.write(idx, UFix(j)) p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } p_rep_cnt_dec_arr.write(UFix(i), flags) } @@ -448,6 +479,8 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) } for( j <- 0 until NTILES ) { From eec369a1c74d019443181b96afb21acfb4eb8e36 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 28 Feb 2012 18:59:15 -0800 Subject: [PATCH 022/374] separate memory request command and data also, merge some VLSI/C++ test harness functionality --- uncore/coherence.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 11c764ec..75c430a3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -424,12 +424,6 @@ class CoherenceHubNoDir extends CoherenceHub { //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd io.mem.req_data <> mem_req_arb.io.out.bits.req_data - //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - // Bits(0, width = MEM_DATA_BITS), - // (0 until NTILES).map( j => - // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - // io.tiles(j).probe_rep_data.bits.data, - // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 6304aa992f92a131c5880d86c0cb659c2aa9a57f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 00:44:03 -0800 Subject: [PATCH 023/374] Fixed race between read resps/reps and write req/reps in null hub --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75c430a3..d7443d71 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -329,19 +329,19 @@ class CoherenceHubNull extends Component { val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready - io.mem.req_cmd.valid := x_init.valid + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid + x_rep.valid := io.mem.resp.valid || is_write } From ef94f13087ab0758cae3958965d746c496e705ea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 02:59:27 -0800 Subject: [PATCH 024/374] Broadcast hub nears completion. Still does not handle generation/arbitration for decoupled mem reqs. --- uncore/coherence.scala | 107 +++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7443d71..75796a9d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -33,9 +33,9 @@ class HubMemReq extends Bundle { val req_data = (new ioDecoupled) { new MemData() } } -class HubProbeRep extends Bundle { - val reply = (new ioDecoupled) { new ProbeReply } - val data_idx = Bits(width = log2up(NTILES)) +class TrackerProbeData extends Bundle { + val valid = Bool() + val data_tile_id = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -180,10 +180,9 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_rep = (new ioDecoupled) { new HubProbeRep() } + val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } @@ -207,13 +206,21 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_READ_UNCACHED) -> P_COPY, (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) } - val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } + def needsMemRead(t_type: UFix, global_state: UFix): Bool = { + (t_type != X_WRITE_UNCACHED) + } + + def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_WRITE_UNCACHED) + } + + val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -223,9 +230,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) - val p_rep_data_idx_ = Reg{ Bits() } - val x_init_data_needs_wb = Reg{ Bool() } - val p_rep_data_needs_wb = Reg{ Bool() } + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg{ Bool() } + val x_init_data_needs_write = Reg{ Bool() } + val p_rep_data_needs_write = Reg{ Bool() } io.busy := state != s_idle io.addr := addr_ @@ -235,15 +243,17 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.t_type := t_type_ io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := Bool(false) io.mem_req.bits.req_cmd.bits.addr := addr_ io.mem_req.bits.req_cmd.bits.tag := UFix(id) + io.mem_req.bits.req_data.valid := Bool(false) + io.mem_req.bits.req_data.bits.data := UFix(0) // := io.mem.ready //sent mem req io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) @@ -258,32 +268,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES) p_req_flags := ~Bits(0, width = NTILES) state := s_probe io.pop_x_init := Bool(true) } } - is(s_mem_r) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_w) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_wr) { - when(io.probe_rep.bits.reply.bits.has_data) { - //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy - //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO - } . otherwise { - //io.pop_x_init := io.mem_req_rdy - //io.pop_x_init_data := io.mem_req_rdy - } - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_mem_r } - } is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags @@ -292,22 +284,55 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } - val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { - state := s_busy //TODO: XXXXXXXXXX + state := s_mem } } - when(p_rep_has_data) { - p_rep_data_needs_wb := Bool(true) - p_rep_data_idx_ := p_rep_data_idx + when(io.probe_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_rep_tile_id } } - is(s_busy) { + is(s_mem) { + when(x_init_data_needs_write) { + //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid + //io.mem_req.bits.req_cmd.valid := // TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.x_init_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_x_init_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (p_rep_data_needs_write) { + //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid + //io.mem_req.bits.req_cmd.valid := //TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.p_rep_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_p_rep_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (x_needs_read) { + io.mem_req.valid := Bool(true) + io.mem_req.bits.req_cmd.valid := Bool(true) + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + x_needs_read := Bool(false) + } + } . otherwise { + io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) + state := s_busy + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } From 8b519e7ea8d578f66597ccd148aa5ec1fa7ee331 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 29 Feb 2012 03:08:04 -0800 Subject: [PATCH 025/374] replace tile memory interface with ioTileLink work in progress towards coherent HTIF. for now, requests are incoherently passed through a null coherence hub. --- uncore/coherence.scala | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75796a9d..9fbd69e9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -10,22 +10,21 @@ class MemData extends Bundle { class MemReqCmd() extends Bundle { val rw = Bool() - val addr = UFix(PADDR_BITS - OFFSET_BITS) - val tag = Bits(MEM_TAG_BITS) + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) } class MemResp () extends Bundle { - val tag = Bits(MEM_TAG_BITS) + val tag = Bits(width = MEM_TAG_BITS) val data = Bits(width = MEM_DATA_BITS) - val valid = Bool() } -class ioMemHub() extends Bundle +class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = new MemResp() + val resp = (new ioValid) { new MemResp() } } class HubMemReq extends Bundle { @@ -49,7 +48,7 @@ class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS) } class TransactionInitData extends MemData @@ -348,8 +347,8 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { - val tile = new ioTileLink() - val mem = new ioMemHub() + val tile = new ioTileLink().flip + val mem = new ioMem } val x_init = io.tile.xact_init @@ -362,11 +361,11 @@ class CoherenceHubNull extends Component { io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid || is_write + x_rep.bits.data := io.mem.resp.bits.data + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write } @@ -388,7 +387,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMemHub + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -427,12 +426,12 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val idx = io.mem.resp.tag + val idx = io.mem.resp.bits.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: From 7e6d990bad5060fbed0dead78e902ba9ce3be04f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 026/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 4f00bcc760ebdd1a41451857f85dc527dbaddf97 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 027/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 940027a888eaf9e9409186a2939ffb3ec5fadafd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 17:58:15 -0800 Subject: [PATCH 028/374] Finished broadcast hub with split mem req types. Untested. --- uncore/coherence.scala | 119 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fbd69e9..f63e633d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -28,8 +28,7 @@ class ioMem() extends Bundle } class HubMemReq extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } + val lock = Bool() } class TrackerProbeData extends Bundle { @@ -187,7 +186,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) @@ -225,14 +226,40 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val probe_done = Reg{ Bits() } - val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg{ Bool() } - val x_init_data_needs_write = Reg{ Bool() } - val p_rep_data_needs_write = Reg{ Bool() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { + req_cmd.valid := mem_cmd_sent + req_cmd.bits.rw := Bool(true) + req_data <> data + lock := Bool(true) + when(req_cmd.ready && req_cmd.valid) { + mem_cmd_sent := Bool(false) + } + when(req_data.ready && req_data.valid) { + pop := Bool(true) + mem_cnt := mem_cnt_next + } + when(mem_cnt === ~UFix(0)) { + trigger := Bool(false) + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready ) { + trigger := Bool(false) + } + } io.busy := state != s_idle io.addr := addr_ @@ -241,14 +268,13 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := Bool(false) - io.mem_req.bits.req_cmd.bits.addr := addr_ - io.mem_req.bits.req_cmd.bits.tag := UFix(id) - io.mem_req.bits.req_data.valid := Bool(false) - io.mem_req.bits.req_data.bits.data := UFix(0) - // := io.mem.ready //sent mem req + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) @@ -269,9 +295,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES) - p_req_flags := ~Bits(0, width = NTILES) - state := s_probe + p_rep_count := UFix(NTILES-1) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) + state := Mux(p_req_flags.orR, s_probe, s_mem) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) } } @@ -285,8 +313,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) state := s_mem } } @@ -296,36 +327,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } } is(s_mem) { - when(x_init_data_needs_write) { - //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid - //io.mem_req.bits.req_cmd.valid := // TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.x_init_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_x_init_data := Bool(true) - //TODO: count with mem_count somehow - } - } . elsewhen (p_rep_data_needs_write) { - //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid - //io.mem_req.bits.req_cmd.valid := //TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.p_rep_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_p_rep_data := Bool(true) - //TODO: count with mem_count somehow - } + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) } . elsewhen (x_needs_read) { - io.mem_req.valid := Bool(true) - io.mem_req.bits.req_cmd.valid := Bool(true) - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - x_needs_read := Bool(false) - } + doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) state := s_busy @@ -369,7 +376,7 @@ class CoherenceHubNull extends Component { } -class CoherenceHubNoDir extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) @@ -440,14 +447,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - - val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready - io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd - io.mem.req_data <> mem_req_arb.io.out.bits.req_data + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 4e33b68beaa90dcc3646c3c305123286427e369b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 01:19:09 -0800 Subject: [PATCH 029/374] Unified hub ios. Fixed some hub elaboration errors. --- uncore/coherence.scala | 52 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f63e633d..0ded6f22 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -222,12 +222,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ Bits() } + val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -350,24 +350,25 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component with CoherencePolicy - -class CoherenceHubNull extends Component { +abstract class CoherenceHub extends Component with CoherencePolicy { val io = new Bundle { - val tile = new ioTileLink().flip + val tiles = Vec(NTILES) { new ioTileLink() }.flip val mem = new ioMem } +} - val x_init = io.tile.xact_init +class CoherenceHubNull extends CoherenceHub { + + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tile.xact_init_data + io.mem.req_data <> io.tiles(0).xact_init_data - val x_rep = io.tile.xact_rep + val x_rep = io.tiles(0).xact_rep x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care @@ -392,13 +393,9 @@ class CoherenceHubBroadcast extends CoherenceHub { ret } - val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem - } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) +/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -467,13 +464,12 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) - val flags = Bits(width = NTILES) + trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) } - p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -484,8 +480,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? @@ -493,7 +489,7 @@ class CoherenceHubBroadcast extends CoherenceHub { x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid - aborting(j) := want_to_abort && x_abort.ready + aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) } // Handle transaction initiation requests @@ -504,10 +500,10 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) + trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data } for( j <- 0 until NTILES ) { @@ -533,15 +529,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } - for( i <- 0 until NGLOBAL_XACTS ) { - val flags = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - flags(j) := p_req_arb_arr(j).io.in(i).ready - } - p_rep_cnt_dec_arr.write(UFix(i), flags) - } - +*/ } From 2152b0283d0a3438b644210166d2122ba64c2580 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 17:03:56 -0800 Subject: [PATCH 030/374] Made xact_rep an ioValid, removed has_data member --- uncore/coherence.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ded6f22..12c1e4a9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -14,10 +14,9 @@ class MemReqCmd() extends Bundle val tag = Bits(width = MEM_TAG_BITS) } -class MemResp () extends Bundle +class MemResp () extends MemData { val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) } class ioMem() extends Bundle @@ -72,7 +71,6 @@ class ProbeReplyData extends MemData class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -88,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep = (new ioValid) { new TransactionReply() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -173,6 +171,10 @@ trait FourStateCoherence extends CoherencePolicy { } state.toBits } + + def replyTypeHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_WRITE_UNCACHED) + } } class XactTracker(id: Int) extends Component with CoherencePolicy { From de73eef4091d41ed41ed824fe65ab19927dfd623 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 18:23:46 -0800 Subject: [PATCH 031/374] Fixed elaboration errors in LockingArbiter and BoradcastHub. Fixed ioDecoupled direction error in XactTracker --- uncore/coherence.scala | 130 ++++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 54 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 12c1e4a9..5f70d81f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -163,13 +163,12 @@ trait FourStateCoherence extends CoherencePolicy { def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { - val state = UFix(0) - switch(incoming.p_type) { - is(probeInvalidate) { state := tileInvalid } - is(probeDowngrade) { state := tileShared } - } - state.toBits + def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state + )) } def replyTypeHasData (reply: TransactionReply): Bool = { @@ -187,9 +186,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val mem_req_data = (new ioDecoupled) { new MemData() }.flip val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) @@ -222,7 +222,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { (t_type === X_WRITE_UNCACHED) } - val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } @@ -241,7 +241,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { req_cmd.valid := mem_cmd_sent req_cmd.bits.rw := Bool(true) - req_data <> data + //TODO: why does req_data <> data segfault? + req_data.valid := data.valid + req_data.bits.data := data.bits.data + data.ready := req_data.ready lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { mem_cmd_sent := Bool(false) @@ -270,7 +273,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -279,7 +282,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) @@ -287,6 +290,8 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) io.send_x_rep_ack := Bool(false) + io.x_init_data.ready := Bool(false) // don't care + io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -336,20 +341,19 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) - state := s_busy + state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) } } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } } } - - //TODO: Decrement the probe count when final data piece is written - // Connent io.mem.ready sig to correct pop* outputs - // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component with CoherencePolicy { @@ -385,19 +389,16 @@ class CoherenceHubBroadcast extends CoherenceHub { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (t_type) { - is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } - is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } - is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } - is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } - } - ret + MuxLookup(t_type, X_READ_UNCACHED, Array( + X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), + X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, + X_READ_UNCACHED -> X_READ_UNCACHED, + X_WRITE_UNCACHED -> X_WRITE_UNCACHED + )) } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) -/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -409,40 +410,61 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) - t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) - trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + do_free_arr(i) := Bool(false) + p_rep_cnt_dec_arr(i) := Bits(0) + p_req_cnt_inc_arr(i) := Bits(0) + sent_x_rep_ack_arr(i) := Bool(false) + t.xact_finish := do_free_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) + t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - do_free_arr.write(finish.bits.global_xact_id, finish.valid) + do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) } // Reply to initial requestor - // Forward memory responses from mem to tile - val idx = io.mem.resp.bits.tag + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.global_xact_id := idx + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.valid := Bool(false) + when(io.mem.resp.valid) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + } io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready - + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data @@ -466,7 +488,8 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -475,14 +498,14 @@ class CoherenceHubBroadcast extends CoherenceHub { } // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } + val aborting = Bits(0, width = NTILES) for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to @@ -505,9 +528,9 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid } - for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data @@ -522,7 +545,6 @@ class CoherenceHubBroadcast extends CoherenceHub { alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) - // Handle probe request generation // Must arbitrate for each request port val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) @@ -535,5 +557,5 @@ class CoherenceHubBroadcast extends CoherenceHub { } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } -*/ + } From 4c3f7a36cedb2f49d657ea1bb151694f66f6b339 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 1 Mar 2012 18:49:00 -0800 Subject: [PATCH 032/374] clean up D$ store data unit --- uncore/coherence.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5f70d81f..2c5c6edf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -90,8 +90,8 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } -trait CoherencePolicy { - def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { +object cpuCmdToRW { + def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool @@ -101,6 +101,9 @@ trait CoherencePolicy { } } +trait CoherencePolicy { +} + trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } From bef59527d6b155b8e888e1d1d57d9dd171903917 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 1 Mar 2012 20:48:46 -0800 Subject: [PATCH 033/374] clean up ioDecoupled/ioPipe interface --- uncore/coherence.scala | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2c5c6edf..97d098bd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = (new ioValid) { new MemResp() } + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() } } class HubMemReq extends Bundle { @@ -80,14 +80,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioValid) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() } + val xact_init_data = (new ioDecoupled) { new TransactionInitData() } + val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() } } object cpuCmdToRW { @@ -181,20 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val mem_req_data = (new ioDecoupled) { new MemData() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 9ad4fd5814cd1578a7aa6de372e77ec5a9e08416 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 12:19:21 -0800 Subject: [PATCH 034/374] BroadcastHub can be elaborated by C and vlsi backends --- uncore/coherence.scala | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 97d098bd..536a4775 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -198,9 +198,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(log2up(NTILES), INPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) @@ -411,8 +411,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } - val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { @@ -424,14 +424,16 @@ class CoherenceHubBroadcast extends CoherenceHub { t_type_arr(i) := t.t_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack - do_free_arr(i) := Bool(false) - p_rep_cnt_dec_arr(i) := Bits(0) - p_req_cnt_inc_arr(i) := Bits(0) - sent_x_rep_ack_arr(i) := Bool(false) t.xact_finish := do_free_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) - t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + for( j <- 0 until NTILES) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } } // Free finished transactions @@ -495,8 +497,7 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } } @@ -556,7 +557,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } From 6ad7aa5cdc586af0ec6a9744f0d6d7cc4cafa1f0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 2 Mar 2012 16:18:32 -0800 Subject: [PATCH 035/374] flip direction of ioPipe to match ioDecoupled --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 536a4775..195234a4 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -23,7 +23,7 @@ class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() } val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() } + val resp = (new ioPipe) { new MemResp() }.flip } class HubMemReq extends Bundle { @@ -86,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val probe_rep = (new ioDecoupled) { new ProbeReply() } val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_rep = (new ioPipe) { new TransactionReply() }.flip val xact_finish = (new ioDecoupled) { new TransactionFinish() } } From 993341d27efcc486234716a1e1c45fedc3cf10fb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 13:41:01 -0800 Subject: [PATCH 036/374] Correction to probe reply w/ data handling --- uncore/coherence.scala | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 195234a4..638cc0d7 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -31,8 +31,7 @@ class HubMemReq extends Bundle { } class TrackerProbeData extends Bundle { - val valid = Bool() - val data_tile_id = Bits(width = log2up(NTILES)) + val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { @@ -182,7 +181,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val probe_data = (new TrackerProbeData).asInput + val p_data = (new ioPipe) { new TrackerProbeData() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) @@ -331,9 +330,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { state := s_mem } } - when(io.probe_data.valid) { + when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_rep_tile_id + p_rep_tile_id_ := io.p_data.bits.tile_id } } is(s_mem) { @@ -413,7 +412,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -425,14 +426,18 @@ class CoherenceHubBroadcast extends CoherenceHub { sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) for( j <- 0 until NTILES) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) } } @@ -490,7 +495,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid From d76f6640070bd34819dc62ddbe8c131ac933ebd0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 21:58:50 -0800 Subject: [PATCH 037/374] Filled out 4 state coherence functions for cache --- uncore/coherence.scala | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 638cc0d7..4532794b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -156,16 +156,26 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.t_type, tileInvalid, Array( + X_READ_SHARED -> tileShared, + X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_READ_UNCACHED -> tileInvalid, + X_WRITE_UNCACHED -> tileInvalid + )) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + } - // TODO: New funcs as compared to incoherent protocol: - def newState(cmd: Bits, state: UFix): UFix - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnPrimaryMiss(cmd: Bits): UFix - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix - - def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - - def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -174,26 +184,26 @@ trait FourStateCoherence extends CoherencePolicy { } def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED) + (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) } } class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val p_data = (new ioPipe) { new TrackerProbeData() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip + val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 7846d5e01d8a1cf86895e56589b98946572cccf9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 23:51:53 -0800 Subject: [PATCH 038/374] Removed has_data fields from all coherence messages, increased message type names to compensate --- uncore/coherence.scala | 78 +++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 4532794b..0ce1f1c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,8 +42,7 @@ class TrackerAllocReq extends Bundle { class TransactionInit extends Bundle { - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() + val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS) } @@ -55,21 +54,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = PTYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = PTYPE_BITS) - val has_data = Bool() + val p_type = Bits(width = P_REP_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = TTYPE_BITS) + val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -162,17 +160,17 @@ trait FourStateCoherence extends CoherencePolicy { } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( - X_READ_SHARED -> tileShared, - X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_READ_UNCACHED -> tileInvalid, - X_WRITE_UNCACHED -> tileInvalid + X_REP_READ_SHARED -> tileShared, + X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_REP_READ_UNCACHED -> tileInvalid, + X_REP_WRITE_UNCACHED -> tileInvalid )) } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { @@ -183,12 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { )) } - def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) + def probeReplyHasData (reply: ProbeReply): Bool = { + (reply.p_type === P_REP_INVALIDATE_DATA || + reply.p_type === P_REP_DOWNGRADE_DATA || + reply.p_type === P_REP_COPY_DATA) + } + def transactionInitHasData (init: TransactionInit): Bool = { + (init.t_type != X_INIT_WRITE_UNCACHED) + } + def transactionReplyHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } } -class XactTracker(id: Int) extends Component with CoherencePolicy { +class XactTracker(id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } @@ -210,7 +216,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) @@ -220,18 +226,20 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, - (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, - (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + MuxLookup(t_type, P_REQ_COPY, Array( + X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, + X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, + X_INIT_READ_UNCACHED -> P_REQ_COPY, + X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + )) } def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_WRITE_UNCACHED) + (t_type != X_INIT_WRITE_UNCACHED) } def needsAckRep(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_WRITE_UNCACHED) + (t_type === X_INIT_WRITE_UNCACHED) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } @@ -312,7 +320,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) @@ -378,7 +386,7 @@ abstract class CoherenceHub extends Component with CoherencePolicy { class CoherenceHubNull extends CoherenceHub { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -387,7 +395,7 @@ class CoherenceHubNull extends CoherenceHub { io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -395,17 +403,17 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_READ_UNCACHED, Array( - X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), - X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, - X_READ_UNCACHED -> X_READ_UNCACHED, - X_WRITE_UNCACHED -> X_WRITE_UNCACHED + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED )) } @@ -415,7 +423,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -506,7 +514,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { @@ -527,7 +535,7 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) + !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? From 802f857cb69bb00086e841ddaddfca135669a8bb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 3 Mar 2012 15:07:22 -0800 Subject: [PATCH 039/374] fix store prefetch bug, it no longer occupies an entry in the sdq --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ce1f1c9..005f17c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -92,8 +92,8 @@ object cpuCmdToRW { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) - val write = store || amo || (cmd === M_PFW) + val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) + val write = store || amo (read, write) } } From a49625c114b5f108451a7c8e64c68cbb36569c34 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 16:34:27 -0800 Subject: [PATCH 040/374] Broadcast hub control logic bugfixes and code cleanup --- uncore/coherence.scala | 81 +++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 005f17c9..c6e702cd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -26,18 +26,13 @@ class ioMem() extends Bundle val resp = (new ioPipe) { new MemResp() }.flip } -class HubMemReq extends Bundle { - val lock = Bool() -} - class TrackerProbeData extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { val xact_init = new TransactionInit() - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() + val tile_id = Bits(width = TILE_ID_BITS) } @@ -98,8 +93,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { -} +trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } @@ -242,36 +236,18 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { - req_cmd.valid := mem_cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { + req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) - //TODO: why does req_data <> data segfault? - req_data.valid := data.valid - req_data.bits.data := data.bits.data data.ready := req_data.ready + req_data <> data lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { - mem_cmd_sent := Bool(false) + cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop := Bool(true) - mem_cnt := mem_cnt_next + pop_data := Bool(true) + mem_cnt := mem_cnt_next } when(mem_cnt === ~UFix(0)) { trigger := Bool(false) @@ -286,6 +262,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ @@ -318,16 +313,16 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.init_tile_id + init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) - state := Mux(p_req_flags.orR, s_probe, s_mem) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) + state := Mux(p_req_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -342,9 +337,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next - when(p_rep_count_next === UFix(0)) { - mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + when(p_rep_count === UFix(0)) { + io.pop_p_rep := Bool(true) state := s_mem } } @@ -355,9 +349,9 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -565,10 +559,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data init_arb.io.in(j).valid := x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.init_tile_id := UFix(j) - init_arb.io.in(j).bits.data_valid := x_init_data.valid - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + init_arb.io.in(j).bits.tile_id := UFix(j) + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && From 82822dfeecd8a5ed3fe9090c3fad56d52ea92dc4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 17:44:30 -0800 Subject: [PATCH 041/374] Added aborted data dequeueing state machine for BroadcastHub --- uncore/coherence.scala | 58 +++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index c6e702cd..fc4f3570 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -240,7 +240,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) data.ready := req_data.ready - req_data <> data + req_data.bits := data.bits + req_data.valid := data.valid lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) @@ -521,23 +522,51 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } // Nack conflicting transaction init attempts - val aborting = Bits(0, width = NTILES) + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) - // Don't abort writebacks stalled on mem. - // TODO: This assumes overlapped writeback init reqs to - // the same addr will never be issued; is this ok? + conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.toBits.andR - x_abort.valid := want_to_abort && x_init.valid - aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(transactionInitHasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + } + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } } // Handle transaction initiation requests @@ -557,15 +586,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data - init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && - !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port From 1024a460f238ff6df738c0d7e119f1ba9d0234be Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 00:31:44 -0800 Subject: [PATCH 042/374] support memory transaction aborts --- uncore/coherence.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc4f3570..7bcd19c3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -395,6 +395,8 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + + io.tiles(0).xact_abort.valid := Bool(false) } From 012b991ffead7aa724adf6565f6ced61e7c9f034 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 15:47:19 -0800 Subject: [PATCH 043/374] implement transaction finish messages --- uncore/coherence.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7bcd19c3..817e6d73 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -397,6 +397,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) } From 3874da4a4ffa304688a0b708322160f48e8b8fd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 13:49:20 -0800 Subject: [PATCH 044/374] Added store dependency queues to BroadcastHub. Minor improvements to utils. --- uncore/coherence.scala | 80 ++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 817e6d73..51505882 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -35,6 +35,9 @@ class TrackerAllocReq extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) @@ -196,9 +199,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } val mem_req_data = (new ioDecoupled) { new MemData } @@ -214,8 +219,10 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_p_rep_dep = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) + val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -236,13 +243,12 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { - req_cmd.valid := !cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - data.ready := req_data.ready + req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - req_data.valid := data.valid - lock := Bool(true) + lock := at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -250,7 +256,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { pop_data := Bool(true) mem_cnt := mem_cnt_next } - when(mem_cnt === ~UFix(0)) { + when(mem_cnt_next === UFix(0)) { + pop_dep := Bool(true) trigger := Bool(false) } } @@ -277,7 +284,6 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) @@ -285,6 +291,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ @@ -303,11 +310,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_p_rep_dep := Bits(0, width = NTILES) io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) + io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) - io.x_init_data.ready := Bool(false) // don't care - io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -321,7 +328,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) state := Mux(p_req_flags.orR, s_probe, s_mem) } @@ -350,9 +358,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + io.pop_p_rep_data, + p_w_mem_cmd_sent, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + io.pop_x_init_data, + x_w_mem_cmd_sent, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -457,6 +481,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } } + val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -510,14 +537,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := do_pop + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -532,14 +568,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { @@ -585,6 +621,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) + trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init @@ -592,8 +631,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 762f2551a70ba40ea8f527522d48551ecc8994a8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 15:54:19 -0800 Subject: [PATCH 045/374] newTransactionOnMiss() --- uncore/coherence.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 51505882..5a52ac44 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -155,6 +155,10 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, From 1e02618cc800bea9c2de5231db628f38ab5ddb62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:01:47 -0800 Subject: [PATCH 046/374] hub code cleanup --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5a52ac44..a04b7096 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -141,6 +141,9 @@ trait FourStateCoherence extends CoherencePolicy { (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) } + //TODO: do we need isPresent() for determining that a line needs to be + //upgraded but that no replacement is needed? + def isValid (state: UFix): Bool = { state != tileInvalid } @@ -224,8 +227,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) - val pop_x_init_data = Bool(OUTPUT) + val pop_x_init = Bits(NTILES, OUTPUT) + val pop_x_init_data = Bits(NTILES, OUTPUT) val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -247,7 +250,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -257,11 +260,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop_data := Bool(true) + pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next } when(mem_cnt_next === UFix(0)) { - pop_dep := Bool(true) + pop_dep := UFix(1) << tile_id trigger := Bool(false) } } @@ -315,8 +318,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bool(false) - io.pop_x_init_data := Bool(false) + io.pop_x_init := Bits(0, width = NTILES) + io.pop_x_init_data := Bits(0, width = NTILES) io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) @@ -334,7 +337,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := Bool(true) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id state := Mux(p_req_flags.orR, s_probe, s_mem) } } @@ -367,20 +370,22 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, - io.pop_p_rep_data, p_w_mem_cmd_sent, + io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, - io.pop_x_init_data, x_w_mem_cmd_sent, + io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -625,23 +630,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) - trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 35ffb80911ec380695fe1629ff695672f00cb33c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:33:11 -0800 Subject: [PATCH 047/374] unified coherence trait functions --- uncore/coherence.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a04b7096..bda67671 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -126,7 +126,15 @@ trait ThreeStateIncoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, state) } - + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, tileClean) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -162,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), From 792960087434d227f164cb63a65579ecfb875a80 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 01:26:35 -0800 Subject: [PATCH 048/374] change D$ to use FourStateCoherence protocol instead of ThreeStateIncoherence. --- uncore/coherence.scala | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bda67671..2b34b8ae 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -121,17 +121,18 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, state) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, tileClean) - } + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) @@ -166,11 +167,20 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -179,11 +189,6 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) - } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( From 1a90e2a1624acbf58ed127c042ffe4847f83792e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 Mar 2012 11:40:49 -0800 Subject: [PATCH 049/374] Broadcast hub bug fixes for load uncached mem req and store uncached xact rep --- uncore/coherence.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2b34b8ae..843ff2be 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,7 +135,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -204,7 +204,7 @@ trait FourStateCoherence extends CoherencePolicy { reply.p_type === P_REP_COPY_DATA) } def transactionInitHasData (init: TransactionInit): Bool = { - (init.t_type != X_INIT_WRITE_UNCACHED) + (init.t_type === X_INIT_WRITE_UNCACHED) } def transactionReplyHasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) @@ -516,12 +516,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + //val ack_idx_ = Reg(ack_idx) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -534,7 +536,6 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.global_xact_id := ack_idx rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) } - io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data } sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: From df6ac34821800d07c9b953b19e5af09a7bc3cd68 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 21:03:44 -0800 Subject: [PATCH 050/374] coherence hub fixes --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 843ff2be..042ee666 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -516,8 +516,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) - //val ack_idx_ = Reg(ack_idx) + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) @@ -534,10 +533,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready @@ -592,13 +591,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Bits(width = NGLOBAL_XACTS) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) + conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 3fc94d627d8f6426433418dccb636d245e1dc204 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 11:36:10 -0800 Subject: [PATCH 051/374] Fixed dependency queue bug in Broadcast Hub --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 042ee666..14504062 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -638,7 +638,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits @@ -655,7 +655,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From ca5caf898c6a26b5074685a4abfdde65d125c8b5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 16:39:05 -0800 Subject: [PATCH 052/374] Hub addr comparison bug fix --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 14504062..e3827c72 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,7 +42,7 @@ class TrackerDependency extends Bundle { class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS - OFFSET_BITS) } class TransactionInitData extends MemData @@ -54,7 +54,7 @@ class TransactionAbort extends Bundle { class ProbeRequest extends Bundle { val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -230,7 +230,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) @@ -449,9 +449,8 @@ class CoherenceHubNull extends CoherenceHub { class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { - addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) - } + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { MuxLookup(t_type, X_REP_READ_UNCACHED, Array( X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), @@ -464,7 +463,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } From 2df5c19f13adf1ae67a9955122c3d6de1f4b51d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 18:07:44 -0800 Subject: [PATCH 053/374] Added require_ack field to TransactionReply bundle --- uncore/coherence.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e3827c72..a2c6ba5f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -68,6 +68,7 @@ class TransactionReply extends MemData { val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() } class TransactionFinish extends Bundle { @@ -440,6 +441,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) @@ -522,6 +524,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) From 2607153b6785875ead46253bc513f86578c0fa53 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 9 Mar 2012 02:08:55 -0800 Subject: [PATCH 054/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 2014db41bd0e826ee5469b8149c6284896a737d2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 9 Mar 2012 11:04:58 -0800 Subject: [PATCH 055/374] Special cased NTILES == 1 due to log2up revision --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a2c6ba5f..275e427b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -297,7 +297,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES-1) + if(NTILES > 1) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -366,7 +366,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - p_rep_count := p_rep_count_next + if(NTILES > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem From d777f1cb24cb951ece5962f79677552a7e899758 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 10 Mar 2012 15:18:57 -0800 Subject: [PATCH 056/374] fix null hub store ack bug --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 275e427b..41959915 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -124,7 +124,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -170,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { } def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -442,7 +442,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) From cb5ce3fe7310c6ae7686665f03bc40a8a8958932 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 14:17:27 -0700 Subject: [PATCH 057/374] More broadcast hub bugfixes --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 41959915..f128036a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -276,17 +276,17 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(req_data.ready && req_data.valid) { pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next - } - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) - when(req_cmd.ready ) { + when(req_cmd.ready) { trigger := Bool(false) } } @@ -535,10 +535,11 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + rep.valid := do_send_ack + sent_x_rep_ack_arr(ack_idx) := do_send_ack } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready From 95f880da704542ea7d346cbb275610ac8cc30ff7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 17:13:01 -0700 Subject: [PATCH 058/374] fixed abort bug --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f128036a..3db3ac46 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -600,7 +600,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 23c822a82e854a70ebc7bd1303e89bd532604600 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Mar 2012 10:38:37 -0700 Subject: [PATCH 059/374] fix hit logic for amos --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3db3ac46..653ab6dc 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -147,8 +147,8 @@ trait FourStateCoherence extends CoherencePolicy { def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) - ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || - (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } //TODO: do we need isPresent() for determining that a line needs to be From 1b733e7cf07c9e8abd95358198bf06d7ade92f48 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 13 Mar 2012 12:34:39 -0700 Subject: [PATCH 060/374] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 1258f31825c7a3e0787784de4fbfe0c86704dfdf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 16:43:35 -0700 Subject: [PATCH 061/374] add probe unit --- uncore/coherence.scala | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 653ab6dc..de158417 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,6 +135,12 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := P_REP_INVALIDATE_ACK + reply.global_xact_id := UFix(0) + reply + } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } @@ -199,6 +205,22 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_DATA, + probeDowngrade -> P_REP_DOWNGRADE_DATA, + probeCopy -> P_REP_COPY_DATA + )) + val without_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_ACK, + probeDowngrade -> P_REP_DOWNGRADE_ACK, + probeCopy -> P_REP_COPY_ACK + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } def probeReplyHasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || @@ -446,6 +468,9 @@ class CoherenceHubNull extends CoherenceHub { io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) } From 53d69d3006b5e52b1b881fcd135eaa8240e734af Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 17:12:01 -0700 Subject: [PATCH 062/374] parameterize broadcast hub by # of tiles --- uncore/coherence.scala | 94 +++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index de158417..ae9321b1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -234,14 +234,14 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -259,13 +259,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bits(NTILES, OUTPUT) - val pop_x_init_data = Bits(NTILES, OUTPUT) - val pop_x_init_dep = Bits(NTILES, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -319,8 +319,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -336,7 +336,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.t_type := t_type_ io.mem_req_cmd.valid := Bool(false) @@ -350,13 +350,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = NTILES) - io.pop_p_rep := Bits(0, width = NTILES) - io.pop_p_rep_data := Bits(0, width = NTILES) - io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bits(0, width = NTILES) - io.pop_x_init_data := Bits(0, width = NTILES) - io.pop_x_init_dep := Bits(0, width = NTILES) + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -368,7 +368,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(NTILES > 1) p_rep_count := UFix(NTILES-1) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -388,7 +388,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(NTILES > 1) p_rep_count := p_rep_count_next + if(ntiles > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem @@ -440,14 +440,14 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } -abstract class CoherenceHub extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() }.flip + val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub { +class CoherenceHubNull extends CoherenceHub(1) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED @@ -474,7 +474,7 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -487,7 +487,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ )) } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -498,8 +498,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } @@ -523,17 +523,17 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ sent_x_rep_ack_arr(i) := Bool(false) p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) @@ -543,7 +543,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -583,7 +583,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -601,10 +601,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -612,9 +612,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } - for( j <- 0 until NTILES ) { + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -662,7 +662,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -671,10 +671,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -694,8 +694,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until NTILES ) { + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 53cd543d3f6ad91c44262225431b1558836256a0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 19:10:54 -0700 Subject: [PATCH 063/374] fix minor coherence bugs --- uncore/coherence.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ae9321b1..fc32b6eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -207,12 +207,12 @@ trait FourStateCoherence extends CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, state, Array( + val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( probeInvalidate -> P_REP_INVALIDATE_DATA, probeDowngrade -> P_REP_DOWNGRADE_DATA, probeCopy -> P_REP_COPY_DATA )) - val without_data = MuxLookup(incoming.p_type, state, Array( + val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( probeInvalidate -> P_REP_INVALIDATE_ACK, probeDowngrade -> P_REP_DOWNGRADE_ACK, probeCopy -> P_REP_COPY_ACK @@ -369,12 +369,13 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_flags.orR, s_probe, s_mem) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -389,7 +390,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(0)) { + when(p_rep_count === UFix(1)) { io.pop_p_rep := Bool(true) state := s_mem } @@ -529,8 +530,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { @@ -589,7 +590,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val idx = p_rep.bits.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := do_pop + p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) From 77c405ffa17d1121439a59d32d6fa38f876d9856 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 14 Mar 2012 06:13:16 -0700 Subject: [PATCH 064/374] use broadcast hub and coherent HTIF --- uncore/coherence.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc32b6eb..7cf36e8a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -237,7 +237,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData } + val p_data = (new ioPipe) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) @@ -387,10 +387,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } when(io.p_rep_cnt_dec.orR) { - val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(1)) { + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { io.pop_p_rep := Bool(true) state := s_mem } @@ -536,7 +536,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS // Free finished transactions for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish - do_free_arr(finish.bits.global_xact_id) := finish.valid + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } finish.ready := Bool(true) } @@ -552,18 +554,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid) { + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx - rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + rep.valid := Bool(true) } . otherwise { rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR - rep.valid := do_send_ack - sent_x_rep_ack_arr(ack_idx) := do_send_ack + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } } } // If there were a ready signal due to e.g. intervening network use: @@ -594,8 +597,10 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) - p_data_tile_id_arr(idx) := UFix(j) + when (p_rep.valid) { + p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + p_data_tile_id_arr(idx) := UFix(j) + } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { From 3129040bda033ae34deffc14f7c4f8f3e0b58f8d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 15 Mar 2012 18:36:07 -0700 Subject: [PATCH 065/374] use divided clk for htif. UDPATE YOUR FESVR by default, we now load programs via a backdoor, because otherwise it takes too long to simulate. --- uncore/slowio.scala | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 uncore/slowio.scala diff --git a/uncore/slowio.scala b/uncore/slowio.scala new file mode 100644 index 00000000..c1535044 --- /dev/null +++ b/uncore/slowio.scala @@ -0,0 +1,49 @@ +package rocket + +import Chisel._ +import Constants._ + +class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +{ + val io = new Bundle { + val out_fast = new ioDecoupled()(data).flip + val out_slow = new ioDecoupled()(data) + + val in_fast = new ioDecoupled()(data) + val in_slow = new ioDecoupled()(data).flip + + val clk_slow = Bool(OUTPUT) + } + + require((divisor & (divisor-1)) == 0) + require(hold_cycles < divisor/2 && hold_cycles >= 2) + + val cnt = Reg() { UFix(width = log2up(divisor)) } + cnt := cnt + UFix(1) + val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time + val in_en = cnt === UFix(divisor/2-1) // rising edge + + val in_slow_rdy = Reg(resetVal = Bool(false)) + val out_slow_val = Reg(resetVal = Bool(false)) + val out_slow_bits = Reg() { data } + + val fromhost_q = new queue(1)(data) + fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.bits := io.in_slow.bits + fromhost_q.io.deq <> io.in_fast + + val tohost_q = new queue(1)(data) + tohost_q.io.enq <> io.out_fast + tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + + when (out_en) { + in_slow_rdy := fromhost_q.io.enq.ready + out_slow_val := tohost_q.io.deq.valid + out_slow_bits := tohost_q.io.deq.bits + } + + io.in_slow.ready := in_slow_rdy + io.out_slow.valid := out_slow_val + io.out_slow.bits := out_slow_bits + io.clk_slow := cnt(log2up(divisor)-1).toBool +} From e38114e4b0f87777a98d3e8e85f8870d3e6bf269 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Mar 2012 01:24:07 -0700 Subject: [PATCH 066/374] fix coherence bug popping wrong store dependence queue --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7cf36e8a..9160a84b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -295,12 +295,14 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && req_data.valid) { - pop_data := UFix(1) << tile_id - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } } @@ -391,7 +393,6 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { - io.pop_p_rep := Bool(true) state := s_mem } } From 5f69c5a764689ee6c85712563100d6b03ffaf5b0 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 20 Mar 2012 00:18:32 -0700 Subject: [PATCH 067/374] fix bug in coherence hub, specifically in abort handling logic --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9160a84b..9cd8fc7c 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -648,9 +648,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - } - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } } } is(s_abort_send) { // nothing is dequeued for now From a7ebea13fcab7fc79b56ad402655531ed8c4650a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 17:03:58 -0700 Subject: [PATCH 068/374] add mem serdes unit --- uncore/memserdes.scala | 89 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 uncore/memserdes.scala diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala new file mode 100644 index 00000000..2305ed69 --- /dev/null +++ b/uncore/memserdes.scala @@ -0,0 +1,89 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ +import scala.math._ + +class ioMemSerialized(w: Int) extends Bundle +{ + val req = (new ioDecoupled) { Bits(width = w) } + val resp = (new ioPipe) { Bits(width = w) }.flip +} + +class MemSerdes(w: Int) extends Component +{ + val io = new Bundle { + val wide = new ioMem().flip + val narrow = new ioMemSerialized(w) + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + val out_buf = Reg() { Bits() } + val in_buf = Reg() { Bits() } + + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_idle) + val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + + when (state === s_idle) { + when (io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) + } + } + when (state === s_read_addr && adone) { + state := s_idle + send_cnt := UFix(0) + } + when (state === s_write_addr && adone) { + state := s_write_idle + send_cnt := UFix(0) + } + when (state === s_write_idle && io.wide.req_data.valid) { + state := s_write_data + } + when (state === s_write_data && ddone) { + data_send_cnt := data_send_cnt + UFix(1) + state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + } + + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + val recv_cnt = Reg() { UFix(width = log2up(rbits)) } + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val resp_val = Reg(resetVal = Bool(false)) + + resp_val := Bool(false) + when (io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + when (recv_cnt === UFix((rbits-1)/w)) { + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + resp_val := Bool(true) + } + in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + } + + io.wide.resp.valid := resp_val + io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) + io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) +} From 5a00143035d1053bd30db2daac4460360c470fa1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 21:45:10 -0700 Subject: [PATCH 069/374] loop host.in to host.out during reset --- uncore/slowio.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index c1535044..395bdc8d 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,7 +28,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val out_slow_bits = Reg() { data } val fromhost_q = new queue(1)(data) - fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -39,7 +39,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp when (out_en) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := tohost_q.io.deq.bits + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 4e6302fedc326e2c95549c10839561e0c5d2759f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 23:03:20 -0700 Subject: [PATCH 070/374] add dessert --- uncore/memserdes.scala | 117 ++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 2305ed69..f68d36e8 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -27,14 +27,28 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) - when (state === s_idle) { - when (io.wide.req_cmd.valid) { - state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) - } + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := send_cnt + UFix(1) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + when (state === s_idle && io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) } when (state === s_read_addr && adone) { state := s_idle @@ -50,26 +64,11 @@ class MemSerdes(w: Int) extends Component when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UFix(1) state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UFix(0) } - when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) - out_buf := out_buf >> UFix(w) - } - when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { - out_buf := io.wide.req_cmd.bits.toBits - } - when (io.wide.req_data.valid && io.wide.req_data.ready) { - out_buf := io.wide.req_data.bits.toBits - } - - io.wide.req_cmd.ready := state === s_idle - io.wide.req_data.ready := state === s_write_idle - io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data - io.narrow.req.bits := out_buf - - val recv_cnt = Reg() { UFix(width = log2up(rbits)) } - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -80,10 +79,80 @@ class MemSerdes(w: Int) extends Component data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } + +class MemDessert(w: Int) extends Component // test rig side +{ + val io = new Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + require(dbits >= abits && rbits >= dbits) + val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_cmd_recv) + + val in_buf = Reg() { Bits() } + when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + } + io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv + + when (state === s_cmd_recv && adone) { + state := s_cmd + recv_cnt := UFix(0) + } + when (state === s_cmd && io.wide.req_cmd.ready) { + state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) + } + when (state === s_data_recv && ddone) { + state := s_data + recv_cnt := UFix(0) + } + when (state === s_data && io.wide.req_data.ready) { + state := s_data_recv + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + data_recv_cnt := data_recv_cnt + UFix(1) + } + when (rdone) { // state === s_reply + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + } + + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + io.wide.req_cmd.valid := state === s_cmd + io.wide.req_cmd.bits.tag := req_cmd + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + + io.wide.req_data.valid := state === s_data + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + + val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + dataq.io.enq <> io.wide.resp + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + + io.narrow.resp.valid := dataq.io.deq.valid + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) +} From 25fe46dc18554e2a3bf78d977a943b64258535c9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 14:18:57 -0700 Subject: [PATCH 071/374] remove bug from dessert --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index f68d36e8..c1d1fccb 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,7 +26,7 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) @@ -98,7 +98,7 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) From 257747a3a149f3f90d9645ab3a2cd3b5ca10b09e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 23:50:09 -0700 Subject: [PATCH 072/374] no dessert tonight :( --- uncore/memserdes.scala | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index c1d1fccb..27bbaa95 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -5,17 +5,17 @@ import Node._ import Constants._ import scala.math._ -class ioMemSerialized(w: Int) extends Bundle +class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = w) } - val resp = (new ioPipe) { Bits(width = w) }.flip + val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip } -class MemSerdes(w: Int) extends Component +class MemSerdes extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val narrow = new ioMemSerialized } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -26,14 +26,14 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -67,19 +67,19 @@ class MemSerdes(w: Int) extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { + when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.wide.resp.valid := resp_val @@ -87,10 +87,10 @@ class MemSerdes(w: Int) extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert(w: Int) extends Component // test rig side +class MemDessert extends Component // test rig side { val io = new Bundle { - val narrow = new ioMemSerialized(w).flip + val narrow = new ioMemSerialized().flip val wide = new ioMem } val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -98,11 +98,11 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -110,7 +110,7 @@ class MemDessert(w: Int) extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -140,19 +140,19 @@ class MemDessert(w: Int) extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) val dataq = (new queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) } From 73d8d42515680d903b18e2a05ecd593dc55d73e9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 9 Apr 2012 21:40:35 -0700 Subject: [PATCH 073/374] fix coherence bug with multiple probe replies --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9cd8fc7c..678f83ab 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -598,8 +598,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid) { - p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + when (p_rep.valid && probeReplyHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) p_data_tile_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) From 27e3346c14778bac96b87ec9ecea7f1118dd781f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 074/374] Refactored coherence better from uncore hub, better coherence function names --- uncore/coherence.scala | 632 ++++++----------------------------------- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+), 544 deletions(-) create mode 100644 uncore/uncore.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 678f83ab..ee51e469 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,42 +3,6 @@ package rocket import Chisel._ import Constants._ -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends Bundle -{ - val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) @@ -75,17 +39,6 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val xact_init_data = (new ioDecoupled) { new TransactionInitData() } - val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish() } -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) @@ -101,6 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, 2) + val X_INIT_READ_EXCLUSIVE = UFix(1, 2) + val X_INIT_WRITE_UNCACHED = UFix(3, 2) + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -116,25 +76,25 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid + def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } reply.p_type := P_REP_INVALIDATE_ACK @@ -151,43 +111,66 @@ trait FourStateCoherence extends CoherencePolicy { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) + val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) + val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) + val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) + + val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) + val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) + val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) + + val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) + val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) + val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) + + def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - - //TODO: do we need isPresent() for determining that a line needs to be - //upgraded but that no replacement is needed? - def isValid (state: UFix): Bool = { state != tileInvalid } - def needsWriteback (state: UFix): Bool = { - state === tileExclusiveDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) - } - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) - } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) - } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -196,8 +179,7 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -205,6 +187,17 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( @@ -221,55 +214,31 @@ trait FourStateCoherence extends CoherencePolicy { reply.global_xact_id := incoming.global_xact_id reply } - def probeReplyHasData (reply: ProbeReply): Bool = { + + def hasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || reply.p_type === P_REP_COPY_DATA) } - def transactionInitHasData (init: TransactionInit): Bool = { + def hasData (init: TransactionInit): Bool = { (init.t_type === X_INIT_WRITE_UNCACHED) } - def transactionReplyHasData (reply: TransactionReply): Bool = { + def hasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } -} -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { - val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + )) } - def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { MuxLookup(t_type, P_REQ_COPY, Array( X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, @@ -281,435 +250,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc def needsMemRead(t_type: UFix, global_state: UFix): Bool = { (t_type != X_INIT_WRITE_UNCACHED) } - - def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { (t_type === X_INIT_WRITE_UNCACHED) } - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue - req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue - req_data.bits := data.bits - lock := at_front_of_dep_queue - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } + def needsAckReply(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_INIT_WRITE_UNCACHED) } } - -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull extends CoherenceHub(1) { - - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ - - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED - )) - } - - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) - } - } - } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && probeReplyHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(transactionInitHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete - } - } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From d301336c3366bc18263ed486d4aa055b85a598ba Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 075/374] Refactored coherence better from uncore hub, better coherence function names --- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 503 insertions(+) create mode 100644 uncore/uncore.scala diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From 6bc47a55b42e28b34cf4845a82387f33cc3c91e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 076/374] changed coherence message type names --- uncore/coherence.scala | 147 ++++++++++++++++++----------------------- uncore/uncore.scala | 28 ++++---- 2 files changed, 78 insertions(+), 97 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ee51e469..7af9c903 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val t_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -29,7 +29,7 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -54,13 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, 2) - val X_INIT_READ_EXCLUSIVE = UFix(1, 2) - val X_INIT_WRITE_UNCACHED = UFix(3, 2) - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val xactInitReadShared = UFix(0, 2) + val xactInitReadExclusive = UFix(1, 2) + val xactInitWriteUncached = UFix(3, 2) + val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -84,57 +84,38 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } - reply.p_type := P_REP_INVALIDATE_ACK + reply.p_type := probeRepInvalidateAck reply.global_xact_id := UFix(0) reply } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) } trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) - val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) - val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) - val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) - - val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) - val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) - val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) - - val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) - val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) - val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) - - - def isHit ( cmd: Bits, state: UFix): Bool = { + def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) @@ -145,8 +126,8 @@ trait FourStateCoherence extends CoherencePolicy { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -171,44 +152,44 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.t_type, tileInvalid, Array( - X_REP_READ_SHARED -> tileShared, - X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_REP_READ_UNCACHED -> tileInvalid, - X_REP_WRITE_UNCACHED -> tileInvalid + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state )) } def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( - probeInvalidate -> P_REP_INVALIDATE_DATA, - probeDowngrade -> P_REP_DOWNGRADE_DATA, - probeCopy -> P_REP_COPY_DATA + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData )) - val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( - probeInvalidate -> P_REP_INVALIDATE_ACK, - probeDowngrade -> P_REP_DOWNGRADE_ACK, - probeCopy -> P_REP_COPY_ACK + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck )) reply.p_type := Mux(needsWriteback(state), with_data, without_data) reply.global_xact_id := incoming.global_xact_id @@ -216,44 +197,44 @@ trait FourStateCoherence extends CoherencePolicy { } def hasData (reply: ProbeReply): Bool = { - (reply.p_type === P_REP_INVALIDATE_DATA || - reply.p_type === P_REP_DOWNGRADE_DATA || - reply.p_type === P_REP_COPY_DATA) + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) } def hasData (init: TransactionInit): Bool = { - (init.t_type === X_INIT_WRITE_UNCACHED) + (init.x_type === xactInitWriteUncached) } def hasData (reply: TransactionReply): Bool = { - (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached )) } - def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { - MuxLookup(t_type, P_REQ_COPY, Array( - X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, - X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, - X_INIT_READ_UNCACHED -> P_REQ_COPY, - X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate )) } - def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_INIT_WRITE_UNCACHED) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) } - def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } - def needsAckReply(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From f7307ee41164fe47fee1a1730c23f8c2929f0c0f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 077/374] changed coherence message type names --- uncore/uncore.scala | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From 17a5d26c1ed11bd1f8adc09a3bf011f0dbf1785c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 078/374] changed coherence type width names to represent max sizes for all protocols --- uncore/coherence.scala | 16 ++++++++-------- uncore/uncore.scala | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7af9c903..be6638aa 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val x_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -16,20 +16,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = P_REQ_TYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_BITS) + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -57,10 +57,10 @@ trait ThreeStateIncoherence extends CoherencePolicy { val xactInitReadShared = UFix(0, 2) val xactInitReadExclusive = UFix(1, 2) val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) + val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From a68f5e016d5a9567095750909582e5b5e2a98153 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 079/374] changed coherence type width names to represent max sizes for all protocols --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From e71e3ce38f16b1b47e386807d0bf8f99b82fa372 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 080/374] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/coherence.scala | 114 ++++++++++++++++++++++++++++------------- uncore/uncore.scala | 4 +- 2 files changed, 79 insertions(+), 39 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index be6638aa..1bd19bbd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,59 +50,99 @@ object cpuCmdToRW { } } -trait CoherencePolicy { } +trait CoherencePolicy { + def isHit (cmd: Bits, state: UFix): Bool + def isValid (state: UFix): Bool -trait ThreeStateIncoherence extends CoherencePolicy { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool + def needsWriteback (state: UFix): Bool + + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnCacheControl(cmd: Bits): UFix + def newStateOnWriteback(): UFix + def newStateOnFlush(): UFix + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits + def getTransactionInitTypeOnWriteback(): Bits + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + + def hasData (reply: ProbeReply): Bool + def hasData (init: TransactionInit): Bool + def hasData (reply: TransactionReply): Bool + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def getTransactionReplyType(x_type: UFix, count: UFix): Bits + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix + def needsMemRead(x_type: UFix, global_state: UFix): Bool + def needsMemWrite(x_type: UFix, global_state: UFix): Bool + def needsAckReply(x_type: UFix, global_state: UFix): Bool +} + +trait IncoherentPolicy extends CoherencePolicy { + // UNIMPLEMENTED + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := UFix(0) + reply.global_xact_id := UFix(0) + reply + } + def hasData (reply: ProbeReply) = Bool(false) + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) +} + +trait ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadShared = UFix(0, 2) - val xactInitReadExclusive = UFix(1, 2) - val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) + val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } + val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } + val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } - def isHit ( cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - ( state === tileClean || state === tileDirty) - } + def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = { - state != tileInvalid - } + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty + def needsWriteback (state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = { - state === tileDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), + xactReplyAck -> tileInvalid + )) + } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, xactInitReadDirty, outstanding.x_type) } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) - } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { - val reply = Wire() { new ProbeReply() } - reply.p_type := probeRepInvalidateAck - reply.global_xact_id := UFix(0) - reply - } - def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO + def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + + def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) } trait FourStateCoherence extends CoherencePolicy { diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From 5acf1d982014bab2c571faf20317ea92ca8f96c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 081/374] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From b22d7f81929b8a01e67306e9e5664b9bdd79d46f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 082/374] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/coherence.scala | 284 +++++++++++++++++++++++++++++++++++++++-- uncore/uncore.scala | 36 +++--- 2 files changed, 289 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 1bd19bbd..e0f5e5c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,7 +50,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { +abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -72,9 +72,10 @@ trait CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply - def hasData (reply: ProbeReply): Bool - def hasData (init: TransactionInit): Bool - def hasData (reply: TransactionReply): Bool + def messageHasData (reply: ProbeReply): Bool + def messageHasData (init: TransactionInit): Bool + def messageHasData (reply: TransactionReply): Bool + def messageUpdatesDataArray (reply: TransactionReply): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -84,7 +85,14 @@ trait CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool } -trait IncoherentPolicy extends CoherencePolicy { +trait UncachedTransactions { + def getTransactionInitTypeOnUncachedRead(): UFix + def getTransactionInitTypeOnUncachedWrite(): UFix +} + +abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions + +abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { @@ -93,7 +101,7 @@ trait IncoherentPolicy extends CoherencePolicy { reply.global_xact_id := UFix(0) reply } - def hasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: ProbeReply) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) @@ -102,7 +110,7 @@ trait IncoherentPolicy extends CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) } -trait ThreeStateIncoherence extends IncoherentPolicy { +class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } @@ -141,11 +149,256 @@ trait ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) - def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -trait FourStateCoherence extends CoherencePolicy { +class TwoStateCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } + val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileValid), Array( + M_INV -> (state === tileValid), + M_CLN -> (state === tileValid) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = state + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileValid + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> tileValid, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI + + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileExclusiveClean, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusiveClean -> xactReplyReadExclusive, + xactInitReadExclusiveDirty -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusiveClean -> probeReqInvalidate, + xactInitReadExclusiveDirty -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class FourStateCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -208,6 +461,8 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -236,17 +491,20 @@ trait FourStateCoherence extends CoherencePolicy { reply } - def hasData (reply: ProbeReply): Bool = { + def messageHasData (reply: ProbeReply): Bool = { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def hasData (init: TransactionInit): Bool = { + def messageHasData (init: TransactionInit): Bool = { (init.x_type === xactInitWriteUncached) } - def hasData (reply: TransactionReply): Bool = { + def messageHasData (reply: TransactionReply): Bool = { (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 1920c97066f6d1a22f6389621a559243fdfc2ab9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 083/374] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/uncore.scala | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 2a7d2888a78c4bac3978b5fb79cd6e6513df7e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 084/374] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 98a5d682a59c3a183145ce85628f58e11e8febcf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 085/374] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 4a6c7dbc26d9fd2197fb7e21367c0cabb920c28a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Apr 2012 17:56:59 -0700 Subject: [PATCH 086/374] Policy determined by constants. MSI policy added. --- uncore/coherence.scala | 147 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e0f5e5c8..b4b23b4b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,7 +154,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -class TwoStateCoherence extends CoherencePolicyWithUncached { +class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } @@ -266,7 +266,7 @@ class TwoStateCoherence extends CoherencePolicyWithUncached { } } -class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI +class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } @@ -398,7 +398,146 @@ class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI } } -class FourStateCoherence extends CoherencePolicyWithUncached { +class MSICoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, (state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveDirty)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> tileExclusiveDirty, + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusive, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -439,7 +578,7 @@ class FourStateCoherence extends CoherencePolicyWithUncached { def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean + M_CLN -> tileShared )) } def newStateOnWriteback() = newStateOnCacheControl(M_INV) From 37eb1a4ae615a8e0d8f94fdff342c8bc38204ec1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 087/374] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From d61e6ee0806466edfc87bc786754290502d4d673 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 088/374] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From 00155f4bc48247b661ffd46831d999d8a35f9918 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 089/374] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From 99bc99f2ad506a8ec5d2e42dbf4447fe19acd9f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 090/374] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From f804c57bb02243004a76c6b7d9b9e6dc38454bfb Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 3 May 2012 04:21:11 -0700 Subject: [PATCH 091/374] reduce HTIF clock divider for now --- uncore/slowio.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 395bdc8d..e54996ef 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -3,7 +3,7 @@ package rocket import Chisel._ import Constants._ -class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { val out_fast = new ioDecoupled()(data).flip @@ -15,8 +15,9 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val clk_slow = Bool(OUTPUT) } + val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 2) + require(hold_cycles < divisor/2 && hold_cycles >= 1) val cnt = Reg() { UFix(width = log2up(divisor)) } cnt := cnt + UFix(1) From 0208e9f95e4283d4971fb8b24d1a8cc5f772254d Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 092/374] removing wires --- uncore/coherence.scala | 10 +++++----- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b4b23b4b..04843fec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -96,7 +96,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() reply.p_type := UFix(0) reply.global_xact_id := UFix(0) reply @@ -209,7 +209,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqCopy -> probeRepCopyData @@ -336,7 +336,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -475,7 +475,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -614,7 +614,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From 6f2f1ba21c573392a7b21f5afa9c863bb9114abe Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 093/374] removing wires --- uncore/uncore.scala | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From f2942f79f98ad17c3fbd423d0896b3a7631b3d21 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 094/374] moving util out into Chisel standard library --- uncore/uncore.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 9b3161920f9a2f0b0905f2df4ee9788a413a3d6e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 095/374] moving util out into Chisel standard library --- uncore/memserdes.scala | 12 ++++++------ uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 27bbaa95..8dc6a8ae 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,8 +26,8 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) @@ -67,8 +67,8 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -98,8 +98,8 @@ class MemDessert extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index e54996ef..0c513849 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -19,7 +19,7 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) require((divisor & (divisor-1)) == 0) require(hold_cycles < divisor/2 && hold_cycles >= 1) - val cnt = Reg() { UFix(width = log2up(divisor)) } + val cnt = Reg() { UFix(width = log2Up(divisor)) } cnt := cnt + UFix(1) val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time val in_en = cnt === UFix(divisor/2-1) // rising edge @@ -46,5 +46,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2up(divisor)-1).toBool + io.clk_slow := cnt(log2Up(divisor)-1).toBool } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 0c6bade592ca5c6216413831d5464b5850e6ec58 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 096/374] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/uncore.scala | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 166b85705533ef7d6d37ea049617b211087f7b1e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 097/374] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/memserdes.scala | 4 ++-- uncore/slowio.scala | 8 ++++---- uncore/uncore.scala | 44 +++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 8dc6a8ae..e20f1ec3 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -7,8 +7,8 @@ import scala.math._ class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip } class MemSerdes extends Component diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 0c513849..6cf5a3d9 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -6,11 +6,11 @@ import Constants._ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { - val out_fast = new ioDecoupled()(data).flip - val out_slow = new ioDecoupled()(data) + val out_fast = new FIFOIO()(data).flip + val out_slow = new FIFOIO()(data) - val in_fast = new ioDecoupled()(data) - val in_slow = new ioDecoupled()(data).flip + val in_fast = new FIFOIO()(data) + val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 66cf6902619c60f9b181c52653b14e1e70b4375a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 098/374] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 1ebfeeca8a0024e2dc2281dc926d1da6ba439e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 099/374] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 0aa33bf9095ddc6b8980a7f12046fbf42bfbf230 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 100/374] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 62a3ea411385f65e8984acc14198b143f0606be1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 101/374] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 18bc14058b3e21ef58b45e59039fd65733f2fd83 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 102/374] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From a79747a06221802ac05c65144d3b8b42d5b7adb2 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 103/374] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From 0258dfb23f28e109dbcd7032e1f5f64aa03d2c5f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 104/374] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From d01e70c6720310386c7c3f8708d24db7798da72d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 105/374] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From c6ac83658130d6c8c80f234aaad939eb80f80875 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 106/374] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From df8aff090672eb388bf165fc85c45e5ebd7fd0fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 107/374] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From d0e12c13f669ff1c1d3c3562dbcfe3bd98366b2d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 108/374] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 77364057267769be338f82a13e0e2c9a223afbb5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 109/374] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 1405718ca8e1a79ffc39c2727ef17a037fcef93f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 110/374] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 1ae3091261f64e1f8e6de7f9e9e2afc4b3c855ce Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 111/374] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 0a2d284d24cda2bd6559975d6ebc823bbea1079c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 112/374] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 465f2efca71727aa952bf157f22ca27ba72d457f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 113/374] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 914b6b622d98646a51437d972f823c4ccab1a27f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 114/374] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 4d4e28c1387b309e9247ba3da94a56b7e5da4dcf Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 115/374] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 85dc34df803dae3d715531a2a85a8c5793acbc62 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 116/374] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 8db233c9b721abf66a47caf2d79c4e5c97c9a8fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 117/374] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 7b9cfd0b90b7f76a5bda02393af873a8e9e9868b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 118/374] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 7a75334bb97d103f7c4ff1c3b936434c5683bebf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 119/374] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 92b7504c9a02c4bd1c067bc71eb1223fd212c05e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 120/374] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From e346f21725e3de2ac2939db11e644f29e9e3ac3b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 121/374] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From 962423d2d150944c6e37e5ccf0b08aac2a05e55f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 122/374] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 875f3622af6520e2985312b07aa0d2ca65732403 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 123/374] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 115c25c34b2f5fe0524b6108e8088c912d5cb428 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 124/374] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From 17dc2075dd2c73b9446eb669a41d45aefd718ff1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 125/374] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From aa7fd1f40b8fb86b48f4d753e3dc5f72cfa7aaaa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 126/374] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/uncore.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From 6546dc84e2e53f5ec4e5b261217981e179f67ce6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 127/374] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/memserdes.scala | 2 +- uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index e20f1ec3..cdd109c2 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -149,7 +149,7 @@ class MemDessert extends Component // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) - val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 6cf5a3d9..068e90c5 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,12 +28,12 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } - val fromhost_q = new queue(1)(data) + val fromhost_q = new Queue(1)(data) fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new queue(1)(data) + val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From fa8075570ab7a5f1fc28ca3c8695dcad5997ef36 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 27 Sep 2012 12:59:45 -0700 Subject: [PATCH 128/374] move srcs into src dir, factoring out uncore consts into consts --- uncore/{ => src}/coherence.scala | 2 +- uncore/src/consts.scala | 28 ++++++++++++++++++++++++++++ uncore/{ => src}/llc.scala | 2 +- uncore/{ => src}/uncore.scala | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) rename uncore/{ => src}/coherence.scala (99%) create mode 100644 uncore/src/consts.scala rename uncore/{ => src}/llc.scala (99%) rename uncore/{ => src}/uncore.scala (99%) diff --git a/uncore/coherence.scala b/uncore/src/coherence.scala similarity index 99% rename from uncore/coherence.scala rename to uncore/src/coherence.scala index 04843fec..e4f85436 100644 --- a/uncore/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala new file mode 100644 index 00000000..8ebf5c49 --- /dev/null +++ b/uncore/src/consts.scala @@ -0,0 +1,28 @@ +package uncore + +import Chisel._ + +object Constants +{ + + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; + + + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 + val TILE_ID_BITS = log2Up(NTILES)+1 + val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + +} diff --git a/uncore/llc.scala b/uncore/src/llc.scala similarity index 99% rename from uncore/llc.scala rename to uncore/src/llc.scala index a2a5d361..2bb2b62a 100644 --- a/uncore/llc.scala +++ b/uncore/src/llc.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ diff --git a/uncore/uncore.scala b/uncore/src/uncore.scala similarity index 99% rename from uncore/uncore.scala rename to uncore/src/uncore.scala index 7ffef155..56acece0 100644 --- a/uncore/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From da6ec486f1e2cfbb0eb501c29262e05d3e52014d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 27 Sep 2012 16:46:36 -0700 Subject: [PATCH 129/374] uncore and rocket changes for new xact types --- uncore/uncore.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7ffef155..84b74366 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -3,14 +3,17 @@ package rocket import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From 2413763f3dfb21acb1c05b1d37ebc34692d1a54e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:05:37 -0700 Subject: [PATCH 130/374] henry's uncore and rocket changes for new xact types --- uncore/src/coherence.scala | 194 +++++++++++++++++++++++++++---------- uncore/src/consts.scala | 60 +++++++++++- uncore/src/uncore.scala | 15 +-- 3 files changed, 205 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e4f85436..7fc03ece 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,10 +3,39 @@ package uncore import Chisel._ import Constants._ -class TransactionInit extends Bundle { +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} +class TransactionInit extends PhysicalAddress { val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS - OFFSET_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) } class TransactionInitData extends MemData @@ -15,10 +44,9 @@ class TransactionAbort extends Bundle { val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } -class ProbeRequest extends Bundle { +class ProbeRequest extends PhysicalAddress { val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -76,6 +104,7 @@ abstract class CoherencePolicy { def messageHasData (init: TransactionInit): Bool def messageHasData (reply: TransactionReply): Bool def messageUpdatesDataArray (reply: TransactionReply): Bool + def messageIsUncached(init: TransactionInit): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -86,8 +115,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getTransactionInitTypeOnUncachedRead(): UFix - def getTransactionInitTypeOnUncachedWrite(): UFix + def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -115,6 +147,8 @@ class ThreeStateIncoherence extends IncoherentPolicy { val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val uncachedTypeList = List() + val hasDataTypeList = List(xactInitWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -149,9 +183,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -159,10 +194,12 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -191,7 +228,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(incoming.x_type, tileInvalid, Array( xactReplyReadExclusive -> tileValid, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -201,8 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached @@ -227,15 +271,14 @@ class MICoherence extends CoherencePolicyWithUncached { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -243,7 +286,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, xactReplyReadUncached, Array( xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -251,7 +297,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, probeReqCopy, Array( xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -271,17 +320,19 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -311,7 +362,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -322,8 +376,12 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) @@ -357,15 +415,14 @@ class MEICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -374,7 +431,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> xactReplyReadExclusive, xactInitReadExclusiveDirty -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -383,7 +443,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> probeReqInvalidate, xactInitReadExclusiveDirty -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -403,10 +466,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -419,7 +484,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -450,7 +515,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> tileExclusiveDirty, xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -461,8 +529,12 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -496,15 +568,14 @@ class MSICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -513,7 +584,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -542,10 +616,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -558,7 +634,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -589,7 +665,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -600,8 +679,12 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -635,15 +718,14 @@ class MESICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -652,7 +734,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -661,7 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> probeReqDowngrade, xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 8ebf5c49..95b07cc8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,15 +1,33 @@ package uncore import Chisel._ +import scala.math._ object Constants { + val NTILES = 1 + val HAVE_RVC = false + val HAVE_FPU = true + val HAVE_VEC = true - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - + val M_X = Bits("b????", 4); + val M_XRD = Bits("b0000", 4); // int load + val M_XWR = Bits("b0001", 4); // int store + val M_PFR = Bits("b0010", 4); // prefetch with intent to read + val M_PFW = Bits("b0011", 4); // prefetch with intent to write + val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines + val M_FENCE = Bits("b0101", 4); // memory fence + val M_INV = Bits("b0110", 4); // write back and invalidate line + val M_CLN = Bits("b0111", 4); // write back line + val M_XA_ADD = Bits("b1000", 4); + val M_XA_SWAP = Bits("b1001", 4); + val M_XA_AND = Bits("b1010", 4); + val M_XA_OR = Bits("b1011", 4); + val M_XA_MIN = Bits("b1100", 4); + val M_XA_MAX = Bits("b1101", 4); + val M_XA_MINU = Bits("b1110", 4); + val M_XA_MAXU = Bits("b1111", 4); + val PADDR_BITS = 40; val VADDR_BITS = 43; val PGIDX_BITS = 13; @@ -18,6 +36,25 @@ object Constants val ASID_BITS = 7; val PERM_BITS = 6; + // rocketNBDCache parameters + val DCACHE_PORTS = 3 + val CPU_DATA_BITS = 64; + val CPU_TAG_BITS = 9; + val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS + val OFFSET_BITS = 6; // log2(cache line size in bytes) + val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses + val NRPQ = 16; // number of secondary misses + val NSDQ = 17; // number of secondary stores/AMOs + val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) + val IDX_BITS = 7; + val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; + val NWAYS = 4 + require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); + + // coherence parameters + val ENABLE_SHARING = true + val ENABLE_CLEAN_EXCLUSIVE = true + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 val TILE_ID_BITS = log2Up(NTILES)+1 @@ -25,4 +62,17 @@ object Constants val NGLOBAL_XACTS = 8 val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val X_INIT_TYPE_MAX_BITS = 2 + val X_INIT_WRITE_MASK_BITS = OFFSET_BITS + val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) + val X_INIT_ATOMIC_OP_BITS = 4 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + // external memory interface + val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS + } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 56acece0..1220a56f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,14 +3,17 @@ package uncore import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From cf8f20584e21560b99970b8edffefa79569dd125 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:08:41 -0700 Subject: [PATCH 131/374] factoring out uncore into separate uncore repo --- uncore/llc.scala | 437 ---------------------------------- uncore/memserdes.scala | 1 + uncore/uncore.scala | 519 ----------------------------------------- 3 files changed, 1 insertion(+), 956 deletions(-) delete mode 100644 uncore/llc.scala delete mode 100644 uncore/uncore.scala diff --git a/uncore/llc.scala b/uncore/llc.scala deleted file mode 100644 index a2a5d361..00000000 --- a/uncore/llc.scala +++ /dev/null @@ -1,437 +0,0 @@ -package rocket - -import Chisel._ -import Node._ -import Constants._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component -{ - class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = new PipeIO()(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = dout1 - } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UFix(width = log2Up(ways)) - val isWriteback = Bool() - - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component -{ - val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - } } - val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) - val request = requests.orR - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int) extends Component -{ - val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMemPipe - } - - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) - } - - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int) extends Component // UNTESTED -{ - val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } - val mem = new ioMem - } - - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } - - val arb = new RRArbiter(n)(new MemReqCmd) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) - for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UFix(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 - when (unlock) { lock := Bool(false) } -} - -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe - } - - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) - - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } - - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 - val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) - - io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data -} diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index cdd109c2..712dec16 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -4,6 +4,7 @@ import Chisel._ import Node._ import Constants._ import scala.math._ +import uncore._ class ioMemSerialized extends Bundle { diff --git a/uncore/uncore.scala b/uncore/uncore.scala deleted file mode 100644 index 84b74366..00000000 --- a/uncore/uncore.scala +++ /dev/null @@ -1,519 +0,0 @@ -package rocket - -import Chisel._ -import Constants._ - -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) - val send_x_rep_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.x_type := x_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } - } -} - -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) -{ - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) -{ - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} From 916c1019af027e7d42133676644f70c4cbb9df06 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 9 Oct 2012 13:02:58 -0700 Subject: [PATCH 132/374] fixed memdessert unpacking --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 712dec16..e8d99514 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -144,8 +144,8 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) From 3973aef9388edfffb1910ea21c7a2e45723be7ab Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 9 Oct 2012 18:04:55 -0700 Subject: [PATCH 133/374] handle structural hazard on LLC tags --- uncore/src/llc.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2bb2b62a..cb3fbb6a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -361,10 +361,10 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val stall_s1 = Bool() val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) replay_s1 := s1_valid && stall_s1 val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) @@ -430,7 +430,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 9610622ab0500addf535474ef91f3de518e5b947 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 12:41:11 -0700 Subject: [PATCH 134/374] moving memserdes + slowio into src --- uncore/{ => src}/memserdes.scala | 0 uncore/{ => src}/slowio.scala | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename uncore/{ => src}/memserdes.scala (100%) rename uncore/{ => src}/slowio.scala (100%) diff --git a/uncore/memserdes.scala b/uncore/src/memserdes.scala similarity index 100% rename from uncore/memserdes.scala rename to uncore/src/memserdes.scala diff --git a/uncore/slowio.scala b/uncore/src/slowio.scala similarity index 100% rename from uncore/slowio.scala rename to uncore/src/slowio.scala From 08ab076217312efd98bf697964e7db7573edfc94 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 15:42:39 -0700 Subject: [PATCH 135/374] forgot to change package + using fromBits in memserdes instead of manual unpacking --- uncore/src/consts.scala | 3 +++ uncore/src/memserdes.scala | 6 ++---- uncore/src/slowio.scala | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 95b07cc8..a1e87f41 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,6 +10,9 @@ object Constants val HAVE_FPU = true val HAVE_VEC = true + val HTIF_WIDTH = 16 + val MEM_BACKUP_WIDTH = HTIF_WIDTH + val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load val M_XWR = Bits("b0001", 4); // int store diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index e8d99514..c44e2543 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ @@ -143,9 +143,7 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd - io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 068e90c5..fa6fef94 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From 1418604bf01252c9b88bf337288b0a303ccf8717 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Oct 2012 18:52:48 -0700 Subject: [PATCH 136/374] new constants organization --- uncore/src/consts.scala | 103 +++++++++++++++++++-------------------- uncore/src/package.scala | 10 ++++ 2 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 uncore/src/package.scala diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index a1e87f41..098bc1d6 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,17 +1,49 @@ package uncore +package constants import Chisel._ -import scala.math._ -object Constants +abstract trait MulticoreConstants { + val NTILES: Int + val TILE_ID_BITS = log2Up(NTILES)+1 +} + +abstract trait CoherenceConfigConstants { + val ENABLE_SHARING: Boolean + val ENABLE_CLEAN_EXCLUSIVE: Boolean +} + +trait UncoreConstants { + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 +} + +trait TileLinkTypeConstants { + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 +} + +trait TileLinkSizeConstants extends + TileLinkTypeConstants { - val NTILES = 1 - val HAVE_RVC = false - val HAVE_FPU = true - val HAVE_VEC = true + val TILE_XACT_ID_BITS = 5 + val X_INIT_WRITE_MASK_BITS = 6 + val X_INIT_SUBWORD_ADDR_BITS = 3 + val X_INIT_ATOMIC_OP_BITS = 4 +} - val HTIF_WIDTH = 16 - val MEM_BACKUP_WIDTH = HTIF_WIDTH +trait MemoryOpConstants { + val MT_X = Bits("b???", 3); + val MT_B = Bits("b000", 3); + val MT_H = Bits("b001", 3); + val MT_W = Bits("b010", 3); + val MT_D = Bits("b011", 3); + val MT_BU = Bits("b100", 3); + val MT_HU = Bits("b101", 3); + val MT_WU = Bits("b110", 3); val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load @@ -30,52 +62,19 @@ object Constants val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); - - val PADDR_BITS = 40; - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; +} - // rocketNBDCache parameters - val DCACHE_PORTS = 3 - val CPU_DATA_BITS = 64; - val CPU_TAG_BITS = 9; - val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS - val OFFSET_BITS = 6; // log2(cache line size in bytes) - val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses - val NRPQ = 16; // number of secondary misses - val NSDQ = 17; // number of secondary stores/AMOs - val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) - val IDX_BITS = 7; - val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; - val NWAYS = 4 - require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); +trait HTIFConstants { + val HTIF_WIDTH = 16 +} - // coherence parameters - val ENABLE_SHARING = true - val ENABLE_CLEAN_EXCLUSIVE = true - - - val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 - val TILE_ID_BITS = log2Up(NTILES)+1 - val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 - val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) - - val X_INIT_TYPE_MAX_BITS = 2 - val X_INIT_WRITE_MASK_BITS = OFFSET_BITS - val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) - val X_INIT_ATOMIC_OP_BITS = 4 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - - // external memory interface +trait MemoryInterfaceConstants extends + HTIFConstants with + UncoreConstants with + TileLinkSizeConstants +{ val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 - val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS - + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_BACKUP_WIDTH = HTIF_WIDTH } diff --git a/uncore/src/package.scala b/uncore/src/package.scala new file mode 100644 index 00000000..17954b54 --- /dev/null +++ b/uncore/src/package.scala @@ -0,0 +1,10 @@ +package uncore +import uncore.constants._ + +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants +{ + +} + From 8509cda8131b896540b1ddf33c879298c42b5870 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 13:58:18 -0700 Subject: [PATCH 137/374] Refined traits for use with rocket asserts, added UncoreConfiguration to handle ntiles --- uncore/src/consts.scala | 21 +++++-- uncore/src/package.scala | 7 ++- uncore/src/uncore.scala | 125 ++++++++++++++++++++------------------- 3 files changed, 85 insertions(+), 68 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 098bc1d6..b7a70bac 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -2,11 +2,7 @@ package uncore package constants import Chisel._ - -abstract trait MulticoreConstants { - val NTILES: Int - val TILE_ID_BITS = log2Up(NTILES)+1 -} +import scala.math.max abstract trait CoherenceConfigConstants { val ENABLE_SHARING: Boolean @@ -19,6 +15,10 @@ trait UncoreConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } +trait CacheConstants extends UncoreConstants { + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + trait TileLinkTypeConstants { val X_INIT_TYPE_MAX_BITS = 2 val X_REP_TYPE_MAX_BITS = 3 @@ -78,3 +78,14 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS val MEM_BACKUP_WIDTH = HTIF_WIDTH } + +trait AddressConstants { + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; +} + diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 17954b54..e3a8f554 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,10 +1,13 @@ package uncore import uncore.constants._ +//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to +// package object uncore and remove import Constants._'s from other files object Constants extends MemoryOpConstants with - MemoryInterfaceConstants + MemoryInterfaceConstants with + CacheConstants with + AddressConstants { - } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 1220a56f..dcd1a819 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -36,13 +36,14 @@ class ioMemPipe() extends Bundle val resp = (new PipeIO) { new MemResp() }.flip } -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { + val tile_id = Bits(width = conf.tile_id_bits) } -class TrackerAllocReq extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) + val tile_id = Bits(width = conf.tile_id_bits) + override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { @@ -61,15 +62,15 @@ class ioTileLink extends Bundle { val incoherent = Bool(OUTPUT) } -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { +class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) + val p_req_cnt_inc = Bits(INPUT, conf.ntiles) + val tile_incoherent = Bits(INPUT, conf.ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -82,18 +83,18 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) + val push_p_req = Bits(OUTPUT, conf.ntiles) + val pop_p_rep = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) + val pop_x_init = Bits(OUTPUT, conf.ntiles) + val pop_x_init_data = Bits(OUTPUT, conf.ntiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -136,8 +137,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -147,15 +148,15 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only + val p_req_initial_flags = Bits(width = conf.ntiles) + p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) @@ -169,13 +170,13 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) + io.push_p_req := Bits(0, width = conf.ntiles) + io.pop_p_rep := Bits(0, width = conf.ntiles) + io.pop_p_rep_data := Bits(0, width = conf.ntiles) + io.pop_p_rep_dep := Bits(0, width = conf.ntiles) + io.pop_x_init := Bits(0, width = conf.ntiles) + io.pop_x_init_data := Bits(0, width = conf.ntiles) + io.pop_x_init_dep := Bits(0, width = conf.ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -192,7 +193,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { + if(conf.ntiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -209,7 +210,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -260,14 +261,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) + +abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip + val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) +class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback @@ -294,23 +297,23 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) } -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) +class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -331,19 +334,19 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -355,7 +358,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -394,7 +397,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -414,10 +417,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -425,9 +428,9 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { + val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ntiles) { Bool() } + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -472,7 +475,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -481,10 +484,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -505,8 +508,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { + val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 9df5cfa5524249d8b1ce3f955e5090cd73e79980 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 14:26:33 -0700 Subject: [PATCH 138/374] Factored out tilelink classes --- uncore/src/coherence.scala | 64 --------------------- uncore/src/tilelink.scala | 111 +++++++++++++++++++++++++++++++++++++ uncore/src/uncore.scala | 45 --------------- 3 files changed, 111 insertions(+), 109 deletions(-) create mode 100644 uncore/src/tilelink.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7fc03ece..dcf384aa 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,70 +3,6 @@ package uncore import Chisel._ import Constants._ -object TransactionInit -{ - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init - } -} -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) -} - -class TransactionInitData extends MemData - -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) -} - -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReplyData extends MemData - -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val require_ack = Bool() -} - -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala new file mode 100644 index 00000000..bddbc0d3 --- /dev/null +++ b/uncore/src/tilelink.scala @@ -0,0 +1,111 @@ +package uncore + +import Chisel._ +import Constants._ + + +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends PhysicalAddress { + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends MemData { + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + +class TransactionInit extends PhysicalAddress { + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +} + +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} + +class TransactionInitData extends MemData + +class TransactionAbort extends Bundle { + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends PhysicalAddress { + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReply extends Bundle { + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends MemData + +class TransactionReply extends MemData { + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() +} + +class TransactionFinish extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new FIFOIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dcd1a819..ebca0834 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,39 +3,6 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val tile_id = Bits(width = conf.tile_id_bits) } @@ -50,18 +17,6 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip From ffda0e41a99e3f344d08e43516ee446d0f28c4b0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:56:36 -0700 Subject: [PATCH 139/374] parameterize width of MemSerdes/MemDesser --- uncore/src/memserdes.scala | 54 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c44e2543..783fb3e1 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -6,17 +6,17 @@ import Constants._ import scala.math._ import uncore._ -class ioMemSerialized extends Bundle +class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = w) } + val resp = (new PipeIO) { Bits(width = w) }.flip } -class MemSerdes extends Component +class MemSerdes(w: Int) extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized + val narrow = new ioMemSerialized(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -27,14 +27,14 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) + out_buf := out_buf >> UFix(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -68,19 +68,19 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { + when (recv_cnt === UFix((rbits-1)/w)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val @@ -88,22 +88,24 @@ class MemSerdes extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert extends Component // test rig side +class MemDesserIO(w: Int) extends Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem +} + +class MemDesser(w: Int) extends Component // test rig side { - val io = new Bundle { - val narrow = new ioMemSerialized().flip - val wide = new ioMem - } + val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -111,7 +113,7 @@ class MemDessert extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -141,17 +143,17 @@ class MemDessert extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) } From 2aecb0024fa12caa8a71aa1250bb25b432e0b8aa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:57:28 -0700 Subject: [PATCH 140/374] UncoreConfiguration now contains coherence policy --- uncore/src/consts.scala | 6 ------ uncore/src/uncore.scala | 16 ++++++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b7a70bac..757959fb 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -64,19 +64,13 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); } -trait HTIFConstants { - val HTIF_WIDTH = 16 -} - trait MemoryInterfaceConstants extends - HTIFConstants with UncoreConstants with TileLinkSizeConstants { val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_BACKUP_WIDTH = HTIF_WIDTH } trait AddressConstants { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ebca0834..80c088f2 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -17,7 +17,8 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { + val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip @@ -216,17 +217,19 @@ class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfigurati } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) -abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub { + val co = conf.co.asInstanceOf[ThreeStateIncoherence] + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp @@ -252,9 +255,10 @@ class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfigura } -class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } From 0cd0f8a9db89d34e057008d051b82a2e48ed6c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 23 Oct 2012 18:01:53 -0700 Subject: [PATCH 141/374] Initial version of migratory protocol --- uncore/src/coherence.scala | 174 ++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index dcf384aa..7bbeaf48 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -9,7 +9,7 @@ object cpuCmdToRW { val load = (cmd === M_XRD) val amo = cmd(3).toBool val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo + val write = store || amo (read, write) } } @@ -556,7 +556,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { @@ -699,3 +699,173 @@ class MESICoherence extends CoherencePolicyWithUncached { (x_type === xactInitWriteUncached) } } + +class MigratoryCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && messageIsUncached(outstanding)) || + (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + tileExclusiveClean -> tileExclusiveDirty, + tileMigratoryClean -> tileMigratoryDirty)), state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( + xactInitReadExclusive -> tileExclusiveDirty, + xactInitReadShared -> tileExclusiveClean)), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid, + xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( + xactInitInvalidateOthers -> tileMigratoryDirty, + xactInitReadExclusive -> tileMigratoryDirty, + xactInitReadShared -> tileMigratoryClean)) + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqInvalidateOthers -> tileInvalid, + probeReqCopy -> state, + probeReqDowngrade -> MuxLookup(state, tileShared, Array( + tileExclusiveClean -> tileSharedByTwo, + tileExclusiveDirty -> tileSharedByTwo, + tileSharedByTwo -> tileShared, + tileMigratoryClean -> tileSharedByTwo, + tileMigratoryDirty -> tileInvalid)) + )) + } + + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new ProbeReply() + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeRepInvalidateDataMigratory, probeRepInvalidateData), + probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + } + def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) + def messageHasData (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + } + def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached, + xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate, + xactInitInvalidateOthers -> probeReqInvalidateOthers + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + } +} From 3e6dc35809509ddffac67a7d4af1d2c0f2294eff Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Nov 2012 02:37:56 -0800 Subject: [PATCH 142/374] issue self-probes for uncached read transactions this facilitates I$ coherence. but it seems like a hack and perhaps the mechanism should be rethought. --- uncore/src/coherence.scala | 15 +++++++-------- uncore/src/consts.scala | 5 ++++- uncore/src/uncore.scala | 34 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7bbeaf48..d7cf1ded 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,14 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = { - val store = (cmd === M_XWR) - val load = (cmd === M_XRD) - val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo - (read, write) - } + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) } abstract class CoherencePolicy { @@ -56,6 +49,7 @@ trait UncachedTransactions { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit + def isUncachedReadTransaction(xinit: TransactionInit): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -182,6 +176,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive @@ -317,6 +312,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -470,6 +466,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -620,6 +617,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -787,6 +785,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 757959fb..57390387 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -50,7 +50,6 @@ trait MemoryOpConstants { val M_XWR = Bits("b0001", 4); // int store val M_PFR = Bits("b0010", 4); // prefetch with intent to read val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines val M_FENCE = Bits("b0101", 4); // memory fence val M_INV = Bits("b0110", 4); // write back and invalidate line val M_CLN = Bits("b0111", 4); // write back line @@ -62,6 +61,10 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); + + def isAMO(cmd: Bits) = cmd(3) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } trait MemoryInterfaceConstants extends diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 80c088f2..e2dcfc30 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -89,10 +89,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } + val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } @@ -105,27 +103,37 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = conf.ntiles) - p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only + p_req_initial_flags := Bits(0) + if (conf.ntiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } io.busy := state != s_idle - io.addr := addr_ + io.addr := xact.addr io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ + io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only - io.x_type := x_type_ + io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.addr := xact.addr io.mem_req_cmd.bits.tag := UFix(id) io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ + io.probe_req.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ntiles) io.pop_p_rep := Bits(0, width = conf.ntiles) io.pop_p_rep_data := Bits(0, width = conf.ntiles) @@ -138,10 +146,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type + xact := io.alloc_req.bits.xact_init init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags @@ -202,7 +208,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { From 6bd4f93f8c27044e6dd6244258b3c4082deda354 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sun, 18 Nov 2012 03:13:17 -0800 Subject: [PATCH 143/374] pull out prefetch commands from isRead --- uncore/src/consts.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 57390387..29be0acc 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -63,7 +63,8 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); def isAMO(cmd: Bits) = cmd(3) - def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW + def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } From 56f9b9721df5283e0fb3f854cf6eb42a52f5bbca Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 20 Nov 2012 05:38:49 -0800 Subject: [PATCH 144/374] treat prefetches as read requests --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d7cf1ded..e8aad9d0 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } abstract class CoherencePolicy { From 8103676b37b40f0a32f46527ad771f3dc0edf54b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Nov 2012 20:54:56 -0800 Subject: [PATCH 145/374] reduce physical address space to 4GB --- uncore/src/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 29be0acc..0b885352 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -78,7 +78,7 @@ trait MemoryInterfaceConstants extends } trait AddressConstants { - val PADDR_BITS = 40; + val PADDR_BITS = 32 val VADDR_BITS = 43; val PGIDX_BITS = 13; val PPN_BITS = PADDR_BITS-PGIDX_BITS; From 50e9d952e874ae5415d87014966a350347ec23e1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 4 Dec 2012 06:57:53 -0800 Subject: [PATCH 146/374] don't initiate llc refill until writeback drains --- uncore/src/llc.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb3fbb6a..2d3eea16 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -109,6 +109,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val requested = Bool() val old_dirty = Bool() val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } @@ -124,12 +125,13 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).way := io.repl_way mshr(freeId).old_dirty := io.repl_dirty mshr(freeId).old_tag := io.repl_tag + mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) mshr(freeId).refillCount := UFix(0) mshr(freeId).refilled := Bool(false) } - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) val request = requests.orR val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -149,7 +151,11 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + when (writeback && io.data.ready && !replay) { + mshr(writebackId).old_dirty := Bool(false) + mshr(writebackId).wb_busy := Bool(true) + } + mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR From aae7a677812e20227d8f6f87b56c40765dbd91f5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 6 Dec 2012 02:07:03 -0800 Subject: [PATCH 147/374] fix llc refill/writeback bugs --- uncore/src/llc.scala | 55 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2d3eea16..7ccdedb8 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -132,7 +132,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component } val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR + val request = requests.orR && io.data.ready // allow in-flight hits to drain val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -365,19 +365,21 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UFix(1) } - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 + val replay_s2 = Reg(resetVal = Bool(false)) + val s2_valid = Reg(resetVal = Bool(false)) + val s2 = Reg() { new MemReqCmd } + val s3_rdy = Bool() + val replay_s2_rdy = Bool() + + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } + when (replay_s2 && replay_s2_rdy) { s1 := s2 } - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } + s2_valid := s1_valid + replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { + when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) @@ -389,26 +391,24 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } + val tag_we = initialize || setDirty || mshr.io.tag.fire() + val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we + tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) + tags.io.in.bits.rw := tag_we + tags.io.in.bits.wdata := Fill(ways, tag_wdata) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty + mshr.io.tag.ready := !s1_valid && !s2_valid data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -419,7 +419,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits @@ -428,15 +428,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 + dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) + s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) + replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we + io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 12caa55dc7e7aa99c4b2449d01fa773d07b8d863 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 148/374] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 8d69f9b41cebb267d5e8cab7756fb2a6e616e10f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 149/374] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From f359518e52375496f6cde523046310b215bc5955 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 150/374] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 6d61baa6cd8d977bae9a93e862bb3972017eb6f2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 151/374] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From 400d48e3de0a00444cef01e03829cfbb2450ced8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 13 Dec 2012 11:39:14 -0800 Subject: [PATCH 152/374] Refactored uncore conf --- uncore/src/coherence.scala | 4 ++ uncore/src/package.scala | 3 +- uncore/src/uncore.scala | 124 ++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e8aad9d0..96fd475d 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,6 +3,10 @@ package uncore import Chisel._ import Constants._ +trait CoherenceAgent +trait ClientCoherenceAgent extends CoherenceAgent +trait MasterCoherenceAgent extends CoherenceAgent + object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a3af64e1..137cc74d 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -35,7 +35,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -63,5 +63,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) + override def clone = { new TileLink().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e2dcfc30..99136c1b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,13 +3,13 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val tile_id = Bits(width = conf.tile_id_bits) +class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { + val tile_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.tile_id_bits) + val tile_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,16 +17,16 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) - val p_req_cnt_inc = Bits(INPUT, conf.ntiles) - val tile_incoherent = Bits(INPUT, conf.ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -39,18 +39,18 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) - val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val init_tile_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) + val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ntiles) - val pop_p_rep = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) - val pop_x_init = Bits(OUTPUT, conf.ntiles) - val pop_x_init_data = Bits(OUTPUT, conf.ntiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) + val push_p_req = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,16 +102,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ntiles) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) p_req_initial_flags := Bits(0) - if (conf.ntiles > 1) { + if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } @@ -120,7 +120,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := xact.tile_xact_id - io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) @@ -134,13 +134,13 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ntiles) - io.pop_p_rep := Bits(0, width = conf.ntiles) - io.pop_p_rep_data := Bits(0, width = conf.ntiles) - io.pop_p_rep_dep := Bits(0, width = conf.ntiles) - io.pop_x_init := Bits(0, width = conf.ntiles) - io.pop_x_init_data := Bits(0, width = conf.ntiles) - io.pop_x_init_dep := Bits(0, width = conf.ntiles) + io.push_p_req := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) + io.pop_x_init := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -155,7 +155,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(conf.ntiles > 1) { + if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -172,7 +172,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -223,16 +223,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) +case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) -abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.ntiles) { new ioTileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co.asInstanceOf[ThreeStateIncoherence] @@ -261,24 +261,24 @@ class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub } -class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -299,19 +299,19 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) + p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -323,7 +323,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -362,7 +362,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -382,10 +382,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -393,9 +393,9 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ntiles) { Bool() } - for( j <- 0 until conf.ntiles ) { + val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -440,7 +440,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -449,10 +449,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -473,8 +473,8 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until conf.ntiles ) { + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 418e3fdf5046111bc34edb00462802ebba20ddc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 7 Jan 2013 13:57:48 -0800 Subject: [PATCH 153/374] Bugfix in crossbar ready sigs, added adapter for old hub, removed incoherent sig from tilelink --- uncore/src/package.scala | 29 +++++++++++++++++---------- uncore/src/uncore.scala | 42 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 137cc74d..9698ffc3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,6 +1,9 @@ package object uncore { import Chisel._ +//TODO: Remove these Networking classes from the package object once Scala bug +//SI-3439 is resolved. + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -22,14 +25,21 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } } - for(i <- 0 until conf.nEndpoints) { + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.header.dst === UFix(i)) + }} + out <> rrarb.io.out + out.header.src := rrarb.io.chosen.toUFix + out.header.dst := UFix(i) + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } @@ -53,7 +63,7 @@ abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalN class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new TileIO) { new TransactionInit } val xact_init_data = (new TileIO) { new TransactionInitData } val xact_abort = (new HubIO) { new TransactionAbort } @@ -62,7 +72,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val probe_rep_data = (new TileIO) { new ProbeReplyData } val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) - override def clone = { new TileLink().asInstanceOf[this.type] } + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 99136c1b..daf756f0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -225,9 +225,47 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) +class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val net = (new TileLinkIO).flip + val hub = Vec(conf.nTiles) { new TileLinkIO } + } + + val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[TileIO[Data]] } + val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[HubIO[Data]] } + + val hubTileProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose + val hubHubProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + + hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { + net.header.src := UFix(0) + net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.valid := hub.map(_.valid).fold(Bool(false))(_||_) + net.bits := hub(0).bits + hub.foreach( _.ready := net.ready) + }} + hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.header := net.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits := net.bits) + net.ready := hub.map(_.ready).fold(Bool(false))(_||_) + }} +} + abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput val mem = new ioMem } } @@ -295,7 +333,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits + t.tile_incoherent := io.incoherent.toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) From f7c01524099f1880ba395e15f9474c50b9d8272c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 15 Jan 2013 15:52:47 -0800 Subject: [PATCH 154/374] Refactored packet headers/payloads --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 92 ++++++++++++++----- uncore/src/tilelink.scala | 2 + uncore/src/uncore.scala | 179 +++++++++++++++++++------------------ 4 files changed, 164 insertions(+), 115 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 96fd475d..caa2b648 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgent -trait ClientCoherenceAgent extends CoherenceAgent -trait MasterCoherenceAgent extends CoherenceAgent +trait CoherenceAgentRole +trait ClientCoherenceAgent extends CoherenceAgentRole +trait MasterCoherenceAgent extends CoherenceAgentRole object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 9698ffc3..3c2a93df 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,5 +1,7 @@ package object uncore { import Chisel._ +import Node._ +import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. @@ -11,32 +13,35 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UFix(width = conf.idBits) } -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asOutput +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data } -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } +} abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} } val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new RRArbiter(conf.nEndpoints)(data) + val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits - rdy := arb.ready && (in.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - out.header.src := rrarb.io.chosen.toUFix - out.header.dst := UFix(i) + //out.bits.header.src := rrarb.io.chosen.toUFix + //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -45,7 +50,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -56,22 +61,63 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UFix(width = conf.idBits) } -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asOutput +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.in.ready := io.out.ready } -class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) -class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } + val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} + val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} + val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} + val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} + val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index a9cbc482..f07e7ed1 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -97,6 +97,7 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } +/* class ioTileLink extends Bundle { val xact_init = (new FIFOIO) { new TransactionInit } val xact_init_data = (new FIFOIO) { new TransactionInitData } @@ -108,3 +109,4 @@ class ioTileLink extends Bundle { val xact_finish = (new FIFOIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) } +*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index daf756f0..697b632c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -4,12 +4,12 @@ import Chisel._ import Constants._ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,7 +17,7 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { +class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip @@ -39,8 +39,8 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) + val init_client_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) @@ -54,7 +54,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { req_cmd.bits.rw := Bool(true) req_data.bits := data.bits when(req_cmd.ready && req_cmd.valid) { @@ -66,11 +66,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { - pop_data := UFix(1) << tile_id + pop_data := UFix(1) << client_id when (data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id + pop_dep := UFix(1) << client_id trigger := Bool(false) } } @@ -90,10 +90,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } - val init_tile_id_ = Reg{ Bits() } + val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_tile_id_ = Reg{ Bits() } + val p_rep_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) val p_rep_data_needs_write = Reg(resetVal = Bool(false)) @@ -111,14 +111,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ + io.init_client_id := init_client_id_ + io.p_rep_client_id := p_rep_client_id_ io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type @@ -147,14 +147,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.xact_init - init_tile_id_ := io.alloc_req.bits.tile_id + init_client_id_ := io.alloc_req.bits.client_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) @@ -179,7 +179,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id + p_rep_client_id_ := io.p_data.bits.client_id } } is(s_mem) { @@ -193,7 +193,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_p_rep_data, io.pop_p_rep_dep, io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) + p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -204,7 +204,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_x_init_data, io.pop_x_init_dep, io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) + init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -231,33 +231,33 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co val hub = Vec(conf.nTiles) { new TileLinkIO } } - val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[TileIO[Data]] } - val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[HubIO[Data]] } + val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } + val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - val hubTileProducedSubBundles = io.hub.map{ io => { + val hubClientProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose - val hubHubProducedSubBundles = io.hub.map{ io => { + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose + val hubMasterProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { - net.header.src := UFix(0) - net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { + net.bits.header.src := UFix(0) + net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.bits.payload := hub(0).bits.payload net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - net.bits := hub(0).bits hub.foreach( _.ready := net.ready) }} - hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.header := net.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits := net.bits) + hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.bits.header := net.bits.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits.payload := net.bits.payload) net.ready := hub.map(_.ready).fold(Bool(false))(_||_) }} } @@ -275,20 +275,20 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren val co = conf.co.asInstanceOf[ThreeStateIncoherence] val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback + val is_write = x_init.bits.payload.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr + io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.payload.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) + x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) + x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) + x_rep.bits.payload.global_xact_id := UFix(0) // don't care + x_rep.bits.payload.data := io.mem.resp.bits.data + x_rep.bits.payload.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) @@ -301,12 +301,13 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { + implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } @@ -316,20 +317,20 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io busy_arr(i) := t.busy addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id + init_client_id_arr(i) := t.init_client_id tile_xact_id_arr(i) := t.tile_xact_id x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits @@ -337,7 +338,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) + p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) @@ -352,7 +353,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -363,28 +364,28 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) + rep.bits.payload.x_type := UFix(0) + rep.bits.payload.tile_xact_id := UFix(0) + rep.bits.payload.global_xact_id := UFix(0) + rep.bits.payload.data := io.mem.resp.bits.data + rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx + when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.payload.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.payload.global_xact_id := ack_idx + when (UFix(j) === init_client_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR sent_x_rep_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -403,29 +404,29 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id + val idx = p_rep.bits.payload.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { + when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) + p_data_client_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) } } @@ -441,16 +442,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { + when(co.messageHasData(x_init.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -485,10 +486,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init @@ -496,11 +497,11 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val x_init_data_dep = x_init_data_dep_list(j).io.deq val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) + init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) @@ -519,7 +520,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req } } From fb2644760ff155c4b4dc14e954d65b5d40157c40 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Jan 2013 23:57:35 -0800 Subject: [PATCH 155/374] single-ported coherence master --- uncore/src/package.scala | 6 +- uncore/src/uncore.scala | 344 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 3c2a93df..2e73a6d7 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -112,11 +112,11 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 697b632c..179179e8 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -524,3 +524,347 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } } + +abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val network = (new TileLinkIO).flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val mem = new ioMem + } +} + +class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +{ + implicit val lnConf = conf.ln + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i) + t.io.tile_incoherent := io.incoherent.toBits + t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) + t.io.mem_resp.bits := io.mem.resp.bits + } + io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val x_init = io.network.xact_init + val x_init_data = io.network.xact_init_data + val x_abort = io.network.xact_abort + val x_dep_deq = x_init_data_dep_q.io.deq + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } + val abort_state = Reg(resetVal = s_idle) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.x_init.bits := x_init.bits + trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + + trackerList(i).io.x_init_data.bits := x_init_data.bits + trackerList(i).io.x_init_data.valid := x_init_data.valid + trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits + trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + } + val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) + x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init + x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) + x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) + x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + + alloc_arb.io.out.ready := x_init.valid + + // Nack conflicting transaction init attempts + x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.valid := Bool(false) + switch(abort_state) { + is(s_idle) { + when(want_to_abort) { + abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { + abort_state := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { // raises x_init.ready + abort_state := s_idle + } + } + } + + // Handle probe request generation + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.p_req.bits + p_req_arb.io.in(i).valid := t.p_req.valid + t.p_req.ready := p_req_arb.io.in(i).ready + } + io.network.probe_req <> p_req_arb.io.out + + // Handle probe replies, which may or may not have data + val p_rep = io.network.probe_rep + val p_rep_data = io.network.probe_rep_data + val idx = p_rep.bits.payload.global_xact_id + p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) + p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) + p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := p_rep_data.valid + trackerList(i).io.p_rep_data.bits := p_rep_data.bits + trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid + trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits + trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) + trackerList(i).io.p_rep.bits := p_rep.bits + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + x_rep_arb.io.in(i).bits := t.x_rep.bits + x_rep_arb.io.in(i).valid := t.x_rep.valid + t.x_rep.ready := x_rep_arb.io.in(i).ready + } + x_rep_arb.io.out.ready := Bool(false) + io.network.xact_rep.valid := x_rep_arb.io.out.valid + io.network.xact_rep.bits := x_rep_arb.io.out.bits + x_rep_arb.io.out.ready := io.network.xact_rep.ready + when(io.mem.resp.valid) { + io.network.xact_rep.valid := Bool(true) + io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + } + } + + // Free finished transactions + val finish = io.network.xact_finish + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + } + finish.ready := Bool(true) + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) +} + +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + val co = conf.co + implicit val ln = conf.ln + val io = new Bundle { + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val free = Bool(INPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val mem_resp = (new FIFOIO) { new MemResp }.flip + + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} + val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val busy = Bool(OUTPUT) + val has_conflict = Bool(OUTPUT) + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new TransactionInit } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + //TODO: Will need id reg for merged release xacts + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) + p_req_initial_flags := Bits(0) + if (conf.ln.nTiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } + val all_x_reps_require_acks = Bool(true) + + io.busy := state != s_idle + io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.p_req.valid := Bool(false) + io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) + io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.addr := xact.addr + io.p_req.bits.header.dst := UFix(0) + io.x_rep.bits.payload.data := io.mem_resp.bits.data + io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) + io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id + io.x_rep.bits.payload.global_xact_id := UFix(id) + io.x_rep.bits.payload.require_ack := all_x_reps_require_acks + io.x_rep.bits.header.dst := init_client_id_ + io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.x_init.ready := Bool(false) + io.x_init_data.ready := Bool(false) + io.x_init_data_dep.ready := Bool(false) + io.p_rep.ready := Bool(false) + io.p_rep_data.ready := Bool(false) + io.p_rep_data_dep.ready := Bool(false) + io.mem_resp.ready := io.x_rep.ready + + switch (state) { + is(s_idle) { + when( io.x_init.valid ) { + xact := io.x_init.bits.payload + init_client_id_ := io.x_init.bits.header.src + init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.x_init.ready := Bool(true) + if(conf.ln.nTiles > 1) { + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } else state := s_mem + } + } + is(s_probe) { + val curr_p_id = PriorityEncoder(p_req_flags) + when(p_req_flags.orR) { + io.p_req.valid := Bool(true) + io.p_req.bits.header.dst := curr_p_id + } + when(io.p_req.ready) { + p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + } + when(io.p_rep.valid) { + io.p_rep.ready := Bool(true) + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) + when(p_rep_count === UFix(1)) { + state := s_mem + } + p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.p_rep_data_dep.ready, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.x_init_data_dep.ready, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, + Mux(all_x_reps_require_acks, s_busy, s_idle)) + } + } + is(s_ack) { + io.x_rep.valid := Bool(true) + when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.free) { + state := s_idle + } + } + } + + def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.bits.rw := Bool(true) + req_data.bits := data.bits.payload + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + data.ready:= Bool(true) + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := Bool(true) + trigger := Bool(false) + } + } + } + } + } + } + + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } +} From c211d74e95ed1561cca1827650b93087b76bcc23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 21 Jan 2013 17:17:26 -0800 Subject: [PATCH 156/374] New TileLink names --- uncore/src/coherence.scala | 898 ++++++++++++++++++------------------- uncore/src/consts.scala | 20 +- uncore/src/package.scala | 16 +- uncore/src/tilelink.scala | 114 ++--- uncore/src/uncore.scala | 286 ++++++------ 5 files changed, 668 insertions(+), 666 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index caa2b648..4268fcf8 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -15,7 +15,7 @@ abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool def needsWriteback (state: UFix): Bool @@ -23,71 +23,71 @@ abstract class CoherencePolicy { def newStateOnCacheControl(cmd: Bits): UFix def newStateOnWriteback(): UFix def newStateOnFlush(): UFix - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix + def newStateOnProbe(incoming: Probe, state: UFix): Bits - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits - def getTransactionInitTypeOnWriteback(): Bits + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getAcquireTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnWriteback(): Bits - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + def newRelease (incoming: Probe, state: UFix): Release - def messageHasData (reply: ProbeReply): Bool - def messageHasData (init: TransactionInit): Bool - def messageHasData (reply: TransactionReply): Bool - def messageUpdatesDataArray (reply: TransactionReply): Bool - def messageIsUncached(init: TransactionInit): Bool + def messageHasData (reply: Release): Bool + def messageHasData (acq: Acquire): Bool + def messageHasData (reply: Grant): Bool + def messageUpdatesDataArray (reply: Grant): Bool + def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool - def getTransactionReplyType(x_type: UFix, count: UFix): Bits - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix - def needsMemRead(x_type: UFix, global_state: UFix): Bool - def needsMemWrite(x_type: UFix, global_state: UFix): Bool - def needsAckReply(x_type: UFix, global_state: UFix): Bool + def getGrantType(a_type: UFix, count: UFix): Bits + def getProbeType(a_type: UFix, global_state: UFix): UFix + def needsMemRead(a_type: UFix, global_state: UFix): Bool + def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsAckReply(a_type: UFix, global_state: UFix): Bool } trait UncachedTransactions { - def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit - def isUncachedReadTransaction(xinit: TransactionInit): Bool + def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire + def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def isUncachedReadTransaction(acq: Acquire): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - reply.p_type := UFix(0) - reply.global_xact_id := UFix(0) + def newStateOnProbe(incoming: Probe, state: UFix): Bits = state + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + reply.r_type := UFix(0) + reply.master_xact_id := UFix(0) reply } - def messageHasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } - val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } - val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } + val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val releaseInvalidateAck :: Nil = Enum(1){ UFix() } val uncachedTypeList = List() - val hasDataTypeList = List(xactInitWriteback) + val hasDataTypeList = List(acquireWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty def needsWriteback (state: UFix): Bool = state === tileDirty @@ -99,28 +99,28 @@ class ThreeStateIncoherence extends IncoherentPolicy { def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), - xactReplyAck -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), + grantAck -> tileInvalid )) } - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) + Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadDirty, outstanding.x_type) + Mux(write, acquireReadDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO - def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO + def getAcquireTypeOnWriteback(): Bits = acquireWriteback - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -128,17 +128,17 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } - val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } - val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), @@ -158,95 +158,95 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> tileValid, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> tileValid, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -255,20 +255,20 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + (write && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -292,108 +292,108 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileExclusiveClean, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileExclusiveClean, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusiveClean -> xactReplyReadExclusive, - xactInitReadExclusiveDirty -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusiveClean -> grantReadExclusive, + acquireReadExclusiveDirty -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusiveClean -> probeReqInvalidate, - xactInitReadExclusiveDirty -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusiveClean -> probeInvalidate, + acquireReadExclusiveDirty -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -402,12 +402,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -418,10 +418,10 @@ class MSICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -445,106 +445,106 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> tileExclusiveDirty, - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> tileExclusiveDirty, + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -553,12 +553,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -569,10 +569,10 @@ class MESICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -596,109 +596,109 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -706,12 +706,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) @@ -723,10 +723,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -752,30 +752,30 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( - xactInitReadExclusive -> tileExclusiveDirty, - xactInitReadShared -> tileExclusiveClean)), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid, - xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( - xactInitInvalidateOthers -> tileMigratoryDirty, - xactInitReadExclusive -> tileMigratoryDirty, - xactInitReadShared -> tileMigratoryClean)) + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( + acquireReadExclusive -> tileExclusiveDirty, + acquireReadShared -> tileExclusiveClean)), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( + acquireInvalidateOthers -> tileMigratoryDirty, + acquireReadExclusive -> tileMigratoryDirty, + acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqInvalidateOthers -> tileInvalid, - probeReqCopy -> state, - probeReqDowngrade -> MuxLookup(state, tileShared, Array( + probeInvalidate -> tileInvalid, + probeInvalidateOthers -> tileInvalid, + probeCopy -> state, + probeDowngrade -> MuxLookup(state, tileShared, Array( tileExclusiveClean -> tileSharedByTwo, tileExclusiveDirty -> tileSharedByTwo, tileSharedByTwo -> tileShared, @@ -784,91 +784,91 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), - probeRepInvalidateDataMigratory, probeRepInvalidateData), - probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new Release() + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + def messageHasData (reply: Release): Bool = { + uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) } - def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) - def messageHasData (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) + def messageHasData (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + def messageUpdatesDataArray (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached, - xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate, - xactInitInvalidateOthers -> probeReqInvalidateOthers + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 0b885352..b515c1c8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -20,19 +20,19 @@ trait CacheConstants extends UncoreConstants { } trait TileLinkTypeConstants { - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 + val ACQUIRE_TYPE_MAX_BITS = 2 + val GRANT_TYPE_MAX_BITS = 3 + val PROBE_TYPE_MAX_BITS = 2 + val RELEASE_TYPE_MAX_BITS = 3 } trait TileLinkSizeConstants extends TileLinkTypeConstants { - val TILE_XACT_ID_BITS = 5 - val X_INIT_WRITE_MASK_BITS = 6 - val X_INIT_SUBWORD_ADDR_BITS = 3 - val X_INIT_ATOMIC_OP_BITS = 4 + val CLIENT_XACT_ID_BITS = 5 + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 } trait MemoryOpConstants { @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 2e73a6d7..0e69ff47 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -110,14 +110,14 @@ class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} - val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} - val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} - val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} - val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index f07e7ed1..8ff26adc 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -32,81 +32,83 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +class Acquire extends PhysicalAddress { + val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -object TransactionInit +object Acquire { - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.write_mask := write_mask + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.subword_addr := subword_addr + acq.atomic_opcode := atomic_opcode + acq } } -class TransactionInitData extends MemData +class AcquireData extends MemData -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +class Abort extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) } -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Probe extends PhysicalAddress { + val p_type = Bits(width = PROBE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Release extends Bundle { + val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReplyData extends MemData +class ReleaseData extends MemData -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Grant extends MemData { + val g_type = Bits(width = GRANT_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) val require_ack = Bool() } -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class GrantAck extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -/* -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 179179e8..690454a1 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -8,13 +8,13 @@ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { - val xact_init = new TransactionInit() + val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { @@ -23,27 +23,27 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) + val grant_ack = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) + val p_rep_data = (new PipeIO) { new ReleaseData }.flip + val x_init_data = (new PipeIO) { new AcquireData }.flip + val sent_grant_ack = Bool(INPUT) val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } + val probe = (new FIFOIO) { new Probe } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) val push_p_req = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) @@ -51,7 +51,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) - val send_x_rep_ack = Bool(OUTPUT) + val send_grant_ack = Bool(OUTPUT) } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { @@ -89,7 +89,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) @@ -108,7 +108,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) @@ -119,9 +119,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.addr := xact.addr io.init_client_id := init_client_id_ io.p_rep_client_id := p_rep_client_id_ - io.tile_xact_id := xact.tile_xact_id + io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only - io.x_type := xact.x_type + io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -130,10 +130,10 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := xact.addr + io.probe.valid := Bool(false) + io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.master_xact_id := UFix(id) + io.probe.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ln.nTiles) io.pop_p_rep := Bits(0, width = conf.ln.nTiles) io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) @@ -141,15 +141,15 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.pop_x_init := Bits(0, width = conf.ln.nTiles) io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) - io.send_x_rep_ack := Bool(false) + io.send_grant_ack := Bool(false) switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.xact_init + xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -164,7 +164,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) + io.probe.valid := Bool(true) } when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs @@ -192,7 +192,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent, io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, @@ -203,20 +203,20 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex x_w_mem_cmd_sent, io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } + io.send_grant_ack := Bool(true) + when(io.sent_grant_ack) { state := s_busy } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { + when (io.grant_ack) { state := s_idle } } @@ -274,28 +274,28 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.payload.x_type === co.xactInitWriteback + val x_init = io.tiles(0).acquire + val is_write = x_init.bits.payload.a_type === co.acquireWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id io.mem.req_cmd.bits.addr := x_init.bits.payload.addr - io.mem.req_data <> io.tiles(0).xact_init_data + io.mem.req_data <> io.tiles(0).acquire_data - val x_rep = io.tiles(0).xact_rep - x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) - x_rep.bits.payload.global_xact_id := UFix(0) // don't care - x_rep.bits.payload.data := io.mem.resp.bits.data - x_rep.bits.payload.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + val grant = io.tiles(0).grant + grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.master_xact_id := UFix(0) // don't care + grant.bits.payload.data := io.mem.resp.bits.data + grant.bits.payload.require_ack := Bool(true) + grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) + io.tiles(0).abort.valid := Bool(false) + io.tiles(0).grant_ack.ready := Bool(true) + io.tiles(0).probe.valid := Bool(false) + io.tiles(0).release.ready := Bool(true) + io.tiles(0).release_data.ready := Bool(true) } @@ -308,15 +308,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } + val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -325,19 +325,19 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co busy_arr(i) := t.busy addr_arr(i) := t.addr init_client_id_arr(i) := t.init_client_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type + client_xact_id_arr(i) := t.client_xact_id + a_type_arr(i) := t.a_type sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) + send_grant_ack_arr(i) := t.send_grant_ack + t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) + sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { @@ -351,9 +351,9 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).xact_finish + val finish = io.tiles(j).grant_ack when (finish.valid) { - do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -361,31 +361,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.payload.x_type := UFix(0) - rep.bits.payload.tile_xact_id := UFix(0) - rep.bits.payload.global_xact_id := UFix(0) + val rep = io.tiles(j).grant + rep.bits.payload.g_type := UFix(0) + rep.bits.payload.client_xact_id := UFix(0) + rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.payload.global_xact_id := mem_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) + rep.bits.payload.master_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.payload.global_xact_id := ack_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) + rep.bits.payload.master_xact_id := ack_idx when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready + rep.valid := send_grant_ack_arr.toBits.orR + sent_grant_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -402,14 +402,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.tiles(j).release + val p_rep_data = io.tiles(j).release_data + val idx = p_rep.bits.payload.master_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) @@ -418,15 +418,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) + val p_rep = io.tiles(j).release + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) } } @@ -435,16 +435,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data + val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -486,23 +486,23 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort + val x_abort = io.tiles(j).abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.acquire := x_init.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) @@ -512,15 +512,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe.bits + p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -552,9 +552,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.xact_init - val x_init_data = io.network.xact_init_data - val x_abort = io.network.xact_abort + val x_init = io.network.acquire + val x_init_data = io.network.acquire_data + val x_abort = io.network.abort val x_dep_deq = x_init_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) @@ -578,14 +578,14 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := x_init.valid // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { @@ -610,23 +610,23 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb.io.in(i).bits := t.p_req.bits p_req_arb.io.in(i).valid := t.p_req.valid t.p_req.ready := p_req_arb.io.in(i).ready } - io.network.probe_req <> p_req_arb.io.out + io.network.probe <> p_req_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.probe_rep - val p_rep_data = io.network.probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.network.release + val p_rep_data = io.network.release_data + val idx = p_rep.bits.payload.master_xact_id p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := p_rep_data.valid @@ -639,29 +639,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - x_rep_arb.io.in(i).bits := t.x_rep.bits - x_rep_arb.io.in(i).valid := t.x_rep.valid - t.x_rep.ready := x_rep_arb.io.in(i).ready + grant_arb.io.in(i).bits := t.grant.bits + grant_arb.io.in(i).valid := t.grant.valid + t.grant.ready := grant_arb.io.in(i).ready } - x_rep_arb.io.out.ready := Bool(false) - io.network.xact_rep.valid := x_rep_arb.io.out.valid - io.network.xact_rep.bits := x_rep_arb.io.out.bits - x_rep_arb.io.out.ready := io.network.xact_rep.ready + grant_arb.io.out.ready := Bool(false) + io.network.grant.valid := grant_arb.io.out.valid + io.network.grant.bits := grant_arb.io.out.bits + grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { - io.network.xact_rep.valid := Bool(true) - io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + io.network.grant.valid := Bool(true) + io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions - val finish = io.network.xact_finish + val finish = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) } finish.ready := Bool(true) @@ -683,10 +683,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip @@ -696,15 +696,15 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} - val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) @@ -730,7 +730,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } - val all_x_reps_require_acks = Bool(true) + val all_grants_require_acks = Bool(true) io.busy := state != s_idle io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) @@ -742,24 +742,24 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) - io.x_rep.bits.payload.data := io.mem_resp.bits.data - io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) - io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id - io.x_rep.bits.payload.global_xact_id := UFix(id) - io.x_rep.bits.payload.require_ack := all_x_reps_require_acks - io.x_rep.bits.header.dst := init_client_id_ - io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.grant.bits.payload.data := io.mem_resp.bits.data + io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.payload.require_ack := all_grants_require_acks + io.grant.bits.header.dst := init_client_id_ + io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) io.x_init_data_dep.ready := Bool(false) io.p_rep.ready := Bool(false) io.p_rep_data.ready := Bool(false) io.p_rep_data_dep.ready := Bool(false) - io.mem_resp.ready := io.x_rep.ready + io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { @@ -768,7 +768,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com init_client_id_ := io.x_init.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -807,7 +807,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com p_rep_data_needs_write, p_w_mem_cmd_sent, io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -816,17 +816,17 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com x_init_data_needs_write, x_w_mem_cmd_sent, io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, - Mux(all_x_reps_require_acks, s_busy, s_idle)) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + Mux(all_grants_require_acks, s_busy, s_idle)) } } is(s_ack) { - io.x_rep.valid := Bool(true) - when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + io.grant.valid := Bool(true) + when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { From b5ccdab514aae4b642c5a88b6c6cbe767420133b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 157/374] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 462 ++++++++++++++++++++-------------------- 1 file changed, 231 insertions(+), 231 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..21aaba8c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -402,31 +402,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -435,31 +435,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -468,7 +468,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -486,41 +486,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -538,8 +538,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -552,49 +552,49 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := acquire.bits.header.src + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -603,38 +603,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -659,11 +659,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -683,20 +683,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -708,32 +708,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -741,11 +741,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.p_req.bits.payload.master_xact_id := UFix(id) - io.p_req.bits.payload.addr := xact.addr - io.p_req.bits.header.dst := UFix(0) + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -753,70 +753,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 1945fa898bb2ab1e242f201112a352eaf4f55dbe Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 24 Jan 2013 23:40:47 -0800 Subject: [PATCH 158/374] make external clock divider programmable --- uncore/src/slowio.scala | 47 ++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index fa6fef94..cf00ae6e 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ import Constants._ -class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { val io = new Bundle { val out_fast = new FIFOIO()(data).flip @@ -13,31 +13,54 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) + + val set_divisor = new PipeIO()(Bits(width = 32)).flip + val divisor = Bits(OUTPUT, 32) } - val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in - require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 1) + require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) + val divisor = Reg(resetVal = UFix(divisor_max-1)) + val d_shadow = Reg(resetVal = UFix(divisor_max-1)) + val hold = Reg(resetVal = UFix(divisor_max/4-1)) + val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + when (io.set_divisor.valid) { + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + } + io.divisor := hold << UFix(16) | divisor - val cnt = Reg() { UFix(width = log2Up(divisor)) } - cnt := cnt + UFix(1) - val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time - val in_en = cnt === UFix(divisor/2-1) // rising edge + val count = Reg{UFix(width = log2Up(divisor_max))} + val clock = Reg{Bool()} + count := count + UFix(1) + + val rising = count === (divisor >> UFix(1)) + val falling = count === divisor + val held = count === (divisor >> UFix(1)) + hold + + when (falling) { + divisor := d_shadow + hold := h_shadow + count := UFix(0) + clock := Bool(false) + } + when (rising) { + clock := Bool(true) + } val in_slow_rdy = Reg(resetVal = Bool(false)) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast - tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val - when (out_en) { + when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) @@ -46,5 +69,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2Up(divisor)-1).toBool + io.clk_slow := clock } From 1134bbf1a4dbc9f479fab240410be8ac0ed1c514 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 27 Jan 2013 11:59:17 -0800 Subject: [PATCH 159/374] cleanup disconnected io pins (overwritten network headers) --- uncore/src/package.scala | 8 +++++--- uncore/src/uncore.scala | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..46d46661 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -62,21 +62,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..01e9c40c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -369,6 +369,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) + rep.bits.header.dst := UFix(0) // DNC + rep.bits.header.src := UFix(0) // DNC rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -445,6 +447,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := UFix(0) // DNC + x_abort.bits.header.src := UFix(0) // DNC want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -585,6 +589,7 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.src := UFix(0) //DNC x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { @@ -746,12 +751,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) + io.p_req.bits.header.src := UFix(0) // DNC io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ + io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) From 47a632cc59cb469458e1a6a8082ea4a0df64d403 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 28 Jan 2013 16:39:45 -0800 Subject: [PATCH 160/374] added support for voluntary wbs over the release network --- uncore/src/coherence.scala | 84 ++++++++++++++++++++--- uncore/src/package.scala | 15 ----- uncore/src/tilelink.scala | 15 ++++- uncore/src/uncore.scala | 135 ++++++++++++++++++++++++++++--------- 4 files changed, 189 insertions(+), 60 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 4268fcf8..e1cc453e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -31,6 +31,7 @@ abstract class CoherencePolicy { def getAcquireTypeOnCacheControl(cmd: Bits): Bits def getAcquireTypeOnWriteback(): Bits + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix): Release def messageHasData (reply: Release): Bool @@ -40,11 +41,15 @@ abstract class CoherencePolicy { def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isVoluntary(rel: Release): Bool def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def requiresAck(grant: Grant): Bool + def requiresAck(release: Release): Bool } trait UncachedTransactions { @@ -70,17 +75,20 @@ abstract class IncoherentPolicy extends CoherencePolicy { def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } val grantData :: grantAck :: Nil = Enum(2){ UFix() } - val releaseInvalidateAck :: Nil = Enum(1){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } val uncachedTypeList = List() val hasDataTypeList = List(acquireWriteback) @@ -106,6 +114,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) @@ -129,9 +140,9 @@ class MICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -180,7 +191,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive @@ -228,6 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, @@ -248,6 +267,8 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MEICoherence extends CoherencePolicyWithUncached { @@ -256,9 +277,9 @@ class MEICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -316,7 +337,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -373,6 +396,12 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -395,6 +424,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MSICoherence extends CoherencePolicyWithUncached { @@ -403,9 +434,9 @@ class MSICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -470,7 +501,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -527,6 +560,12 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -546,6 +585,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MESICoherence extends CoherencePolicyWithUncached { @@ -554,9 +595,9 @@ class MESICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -621,7 +662,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -678,6 +721,12 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -700,6 +749,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -707,9 +759,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -789,7 +841,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -848,6 +902,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -871,4 +931,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..8e511663 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -105,19 +105,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 8ff26adc..51b64b16 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -79,8 +79,20 @@ class Probe extends PhysicalAddress { val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class Release extends Bundle { +object Release +{ + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel.client_xact_id := client_xact_id + rel.master_xact_id := master_xact_id + rel + } +} +class Release extends PhysicalAddress { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } @@ -90,7 +102,6 @@ class Grant extends MemData { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) - val require_ack = Bool() } class GrantAck extends Bundle { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 21aaba8c..a439c7b6 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -288,7 +288,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data - grant.bits.payload.require_ack := Bool(true) grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) @@ -368,7 +367,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.client_xact_id := UFix(0) rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data - rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -537,11 +535,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i) t.io.tile_incoherent := io.incoherent.toBits t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) @@ -559,12 +557,12 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) + val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - for( i <- 0 until NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } + for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready @@ -610,8 +608,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 until NGLOBAL_XACTS ) { + val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb.io.in(i).bits := t.probe.bits probe_arb.io.in(i).valid := t.probe.valid @@ -622,13 +620,16 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle probe replies, which may or may not have data val release = io.network.release val release_data = io.network.release_data - val idx = release.bits.payload.master_xact_id + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} + val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) release.ready := trackerList.map(_.io.release.ready).reduce(_||_) release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.release_data.valid := release_data.valid trackerList(i).io.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid @@ -639,28 +640,27 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 until NGLOBAL_XACTS ) { + val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io grant_arb.io.in(i).bits := t.grant.bits grant_arb.io.in(i).valid := t.grant.valid t.grant.ready := grant_arb.io.in(i).ready } - grant_arb.io.out.ready := Bool(false) io.network.grant.valid := grant_arb.io.out.valid io.network.grant.bits := grant_arb.io.out.bits grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { io.network.grant.valid := Bool(true) io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions val ack = io.network.grant_ack - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } ack.ready := Bool(true) @@ -668,9 +668,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } + for( i <- 0 to NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock @@ -679,7 +679,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren io.mem.req_data <> Queue(mem_req_data_arb.io.out) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + +abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { @@ -699,15 +700,83 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) - val has_conflict = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } +} +class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new Release } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val release_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.mem_resp.ready := Bool(false) + io.probe.valid := Bool(false) + io.busy := Bool(true) + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) + io.grant.valid := Bool(false) + io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.header.dst := init_client_id_ + + switch (state) { + is(s_idle) { + when( io.release.valid ) { + xact := io.release.bits.payload + init_client_id_ := io.release.bits.header.src + release_data_needs_write := co.messageHasData(io.release.bits.payload) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) + io.release.ready := Bool(true) + state := s_mem + } + } + is(s_mem) { + when (release_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.release_data, + release_data_needs_write, + mem_cmd_sent, + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . otherwise { state := s_ack } + } + is(s_ack) { + io.grant.valid := Bool(true) + when(io.grant.ready) { state := s_idle } + } + } +} + +class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) @@ -715,9 +784,6 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { @@ -730,10 +796,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -747,10 +813,9 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.probe.bits.payload.addr := xact.addr io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.g_type := grant_type io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.acquire.ready := Bool(false) @@ -821,12 +886,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(all_grants_require_acks, s_busy, s_idle)) + Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } + when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { @@ -834,6 +899,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } } } +} + +trait MemoryRequestGenerator { + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { req_cmd.bits.rw := Bool(true) From 944f56a7660166933e74cbc4942d84e392905601 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 28 Feb 2013 14:55:19 -0800 Subject: [PATCH 161/374] remove duplicate definitions --- uncore/src/package.scala | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46d46661..21ea9433 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -106,20 +106,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon val payload = data override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } - -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } From 9f0ccbeac58c32ae2416e030d53dc656969fb7a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 18:13:41 -0800 Subject: [PATCH 162/374] writebacks on release network pass asm tests and bmarks --- uncore/src/coherence.scala | 213 +++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 113 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e1cc453e..effba2e9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -28,20 +28,21 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getAcquireTypeOnCacheControl(cmd: Bits): Bits - def getAcquireTypeOnWriteback(): Bits + def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getReleaseTypeOnVoluntaryWriteback(): Bits def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix): Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (reply: Release): Bool + def messageHasData (rel: Release): Bool def messageHasData (acq: Acquire): Bool - def messageHasData (reply: Grant): Bool + def messageHasData (grant: Grant): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool + def isVoluntary(gnt: Grant): Bool def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix @@ -50,6 +51,9 @@ abstract class CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -66,13 +70,8 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release - reply.r_type := UFix(0) - reply.master_xact_id := UFix(0) - reply - } - def messageHasData (reply: Release) = Bool(false) + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -82,15 +81,19 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } - val uncachedTypeList = List() - val hasDataTypeList = List(acquireWriteback) + val uncachedAcquireTypeList = List() + val hasDataAcquireTypeList = List(acquireWriteback) + val hasDataReleaseTypeList = List(acquireWriteback) + val hasDataGrantTypeList = List(grantData) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -116,6 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -125,13 +129,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO - def getAcquireTypeOnWriteback(): Bits = acquireWriteback + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -143,8 +147,10 @@ class MICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -194,14 +200,14 @@ class MICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,23 +216,16 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -269,6 +268,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -280,8 +280,11 @@ class MEICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -340,6 +343,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -349,11 +353,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -364,24 +367,16 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -426,6 +421,8 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -437,8 +434,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -504,6 +503,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -513,11 +513,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -528,24 +527,16 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -587,6 +578,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -598,8 +591,11 @@ class MESICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -665,6 +661,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -674,11 +671,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -689,24 +685,16 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -752,6 +740,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -762,10 +752,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -844,6 +835,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -853,12 +845,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new Release() val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), @@ -871,22 +862,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) - } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) - def messageHasData (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) - } + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -933,4 +918,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } From 717a78f96435b51de9e99a2b9d0f3f3964d93fda Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:09 -0700 Subject: [PATCH 163/374] fix seqRead inference --- uncore/src/llc.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 7ccdedb8..55d543b7 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -40,7 +40,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) @@ -48,15 +48,15 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } + .otherwise { if (postLatency > 0) ridx := idx } } if (postLatency == 0) { dout = mem(idx) } else if (postLatency == 1) { - dout = dout1 + dout = mem(ridx) } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits rdata(j) := dout } From f120800aa259592400793f7dfa1b15a141f82e6c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:28 -0700 Subject: [PATCH 164/374] add DRAMSideLLCNull --- uncore/src/llc.scala | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 55d543b7..1f9f4b2a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -442,3 +442,99 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } + +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(resetVal = Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + do_flow := empty && io.deq.ready + + val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} + val ram_addr = Reg{Bits()} + val ram_out_valid = Reg{Bool()} + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + + val fq = new HellaFlowQueue(entries)(data) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { + val q = (new HellaQueue(entries)) { enq.bits.clone } + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMemPipe + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(resetVal = UFix(numEntries, size)) + val watermark = count >= UFix(refillCycles) + + when (inc && !dec) { + count := count + UFix(1) + } + when (!inc && dec) { + count := count - UFix(refillCycles) + } + when (inc && dec) { + count := count - UFix(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} From 7b019cb0da4f2699e9b1fc45b8f21fc7adf9c96d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 19 Mar 2013 15:30:23 -0700 Subject: [PATCH 165/374] rmeove aborts --- uncore/src/tilelink.scala | 5 -- uncore/src/uncore.scala | 96 ++++----------------------------------- 2 files changed, 8 insertions(+), 93 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 51b64b16..73bf8aff 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -70,10 +70,6 @@ object Acquire class AcquireData extends MemData -class Abort extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) -} - class Probe extends PhysicalAddress { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) @@ -115,7 +111,6 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index a439c7b6..8d1d396d 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -290,7 +290,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.data := io.mem.resp.bits.data grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) io.tiles(0).probe.valid := Bool(false) io.tiles(0).release.ready := Bool(true) @@ -427,51 +426,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } - for( j <- 0 until conf.ln.nTiles ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val x_abort = io.tiles(j).abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) - } - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(acquire.bits.payload)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready below - abort_state_arr(j) := s_idle - } - } - } - } // Handle transaction initiation requests // Only one allocation per cycle @@ -493,16 +447,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq - val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).valid := acquire.valid init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - val do_pop = foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire.ready := foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire.ready := (x_abort.valid && x_abort.ready) || do_pop - acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } @@ -552,61 +504,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Init requests may or may not have data val acquire = io.network.acquire val acquire_data = io.network.acquire_data - val x_abort = io.network.abort val x_dep_deq = acquire_data_dep_q.io.deq - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state = Reg(resetVal = s_idle) - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready trackerList(i).io.acquire_data.bits := acquire_data.bits trackerList(i).io.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire - acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid - // Nack conflicting transaction init attempts - x_abort.bits.header.dst := acquire.bits.header.src - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - x_abort.valid := Bool(false) - switch(abort_state) { - is(s_idle) { - when(want_to_abort) { - abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready - abort_state := s_idle - } - } - } - // Handle probe request generation val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 to NGLOBAL_XACTS ) { From a7ae7e5758eea704ed284667df0295e9b304a30c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 19:49:05 -0800 Subject: [PATCH 166/374] Cleaned up self-probes --- uncore/src/coherence.scala | 7 +++++++ uncore/src/uncore.scala | 11 +++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index effba2e9..f7a75afc 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -49,6 +49,7 @@ abstract class CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool @@ -79,6 +80,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) @@ -268,6 +270,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -421,6 +424,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -578,6 +582,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -740,6 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -918,6 +924,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 8d1d396d..e3ebbc07 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -105,14 +105,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) + val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } io.busy := state != s_idle From 319b4544d757f871a9fcd6437abf62847eeab179 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 21:03:56 -0800 Subject: [PATCH 167/374] nTiles -> nClients in LogicalNetworkConfig --- uncore/src/package.scala | 2 +- uncore/src/uncore.scala | 116 +++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 8e511663..7cce3ff3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -48,7 +48,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e3ebbc07..5609b701 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,9 +24,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nClients) + val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data = (new PipeIO) { new ReleaseData }.flip val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) @@ -44,13 +44,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nTiles) - val pop_release = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nClients) + val pop_release = Bits(OUTPUT, conf.ln.nClients) + val pop_release_data = Bits(OUTPUT, conf.ln.nClients) + val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) val send_grant_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,11 +102,11 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } @@ -115,7 +115,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.init_client_id := init_client_id_ io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) @@ -129,13 +129,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nTiles) - io.pop_release := Bits(0, width = conf.ln.nTiles) - io.pop_release_data := Bits(0, width = conf.ln.nTiles) - io.pop_release_dep := Bits(0, width = conf.ln.nTiles) - io.pop_acquire := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nClients) + io.pop_release := Bits(0, width = conf.ln.nClients) + io.pop_release_data := Bits(0, width = conf.ln.nClients) + io.pop_release_dep := Bits(0, width = conf.ln.nClients) + io.pop_acquire := Bits(0, width = conf.ln.nClients) + io.pop_acquire_data := Bits(0, width = conf.ln.nClients) + io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) io.send_grant_ack := Bool(false) switch (state) { @@ -150,7 +150,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -167,7 +167,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when(io.release_cnt_dec.orR) { val dec = PopCount(io.release_cnt_dec) io.pop_release := io.release_cnt_dec - if(conf.ln.nTiles > 1) release_count := release_count - dec + if(conf.ln.nClients > 1) release_count := release_count - dec when(release_count === dec) { state := s_mem } @@ -223,7 +223,7 @@ case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConf class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val net = (new TileLinkIO).flip - val hub = Vec(conf.nTiles) { new TileLinkIO } + val hub = Vec(conf.nClients) { new TileLinkIO } } val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => @@ -259,8 +259,8 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val tiles = Vec(conf.nClients) { new TileLinkIO }.flip + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -307,8 +307,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -333,17 +333,17 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { release_cnt_dec_arr(i)(j) := Bool(false) probe_cnt_inc_arr(i)(j) := Bool(false) } } - val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val ack = io.tiles(j).grant_ack when (ack.valid) { do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) @@ -355,7 +355,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val rep = io.tiles(j).grant rep.bits.payload.g_type := UFix(0) rep.bits.payload.client_xact_id := UFix(0) @@ -393,7 +393,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val release = io.tiles(j).release val release_data = io.tiles(j).release_data val idx = release.bits.payload.master_xact_id @@ -413,10 +413,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { val release = io.tiles(j).release release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } @@ -426,7 +426,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -435,10 +435,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq @@ -457,8 +457,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nTiles ) { + val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + for( j <- 0 until conf.ln.nClients ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb_arr(j).io.in(i).bits := t.probe.bits @@ -473,7 +473,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val network = (new TileLinkIO).flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -604,7 +604,7 @@ abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) ex val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip @@ -624,7 +624,7 @@ class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extend val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) @@ -688,27 +688,27 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -746,7 +746,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends when( io.acquire.valid ) { xact := io.acquire.bits.payload init_client_id_ := io.acquire.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags @@ -754,7 +754,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.acquire.ready := Bool(true) - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -771,7 +771,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends } when(io.release.valid) { io.release.ready := Bool(true) - if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } From c36b1dfa30290b5009925c719c99e243a7f6313a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 20 Mar 2013 14:10:16 -0700 Subject: [PATCH 168/374] Cleaned up uncore and coherence interface. Removed defunct broadcast hub. Trait-ified tilelink bundle components. Added generalized mem arbiter. --- uncore/src/coherence.scala | 86 ++-- uncore/src/consts.scala | 6 +- uncore/src/package.scala | 10 +- uncore/src/tilelink.scala | 132 +++++- uncore/src/uncore.scala | 841 ++++++++----------------------------- 5 files changed, 345 insertions(+), 730 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index f7a75afc..b4c4067f 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -34,9 +34,7 @@ abstract class CoherencePolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (rel: Release): Bool - def messageHasData (acq: Acquire): Bool - def messageHasData (grant: Grant): Bool + def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool @@ -46,8 +44,8 @@ abstract class CoherencePolicy { def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix - def needsMemRead(a_type: UFix, global_state: UFix): Bool - def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UFix, global_state: UFix): Bool + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool @@ -72,13 +70,12 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) - def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) @@ -134,8 +131,12 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData( msg: SourcedMessage ) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => Bool(false) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } @@ -221,9 +222,12 @@ class MICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -259,10 +263,10 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -373,9 +377,12 @@ class MEICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -413,10 +420,10 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -534,9 +541,12 @@ class MSICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -571,10 +581,10 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -693,9 +703,12 @@ class MESICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -733,10 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -871,9 +884,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } @@ -913,10 +929,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b515c1c8..d348e0ff 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -29,7 +29,7 @@ trait TileLinkTypeConstants { trait TileLinkSizeConstants extends TileLinkTypeConstants { - val CLIENT_XACT_ID_BITS = 5 + val CLIENT_XACT_ID_MAX_BITS = 10 val ACQUIRE_WRITE_MASK_BITS = 6 val ACQUIRE_SUBWORD_ADDR_BITS = 3 val ACQUIRE_ATOMIC_OP_BITS = 4 @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7cce3ff3..492079cc 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -40,8 +40,6 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - //out.bits.header.src := rrarb.io.chosen.toUFix - //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -62,21 +60,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 73bf8aff..c16f8c9b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,20 +3,30 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { +trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -class MemData extends Bundle { +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +} + +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +} + +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends PhysicalAddress { +class MemData extends Bundle with HasMemData + +class MemReqCmd extends Bundle with HasPhysicalAddress { val rw = Bool() val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends MemData { +class MemResp extends Bundle with HasMemData { val tag = Bits(width = MEM_TAG_BITS) } @@ -32,9 +42,12 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class Acquire extends PhysicalAddress { +trait SourcedMessage extends Bundle +trait ClientSourcedMessage extends SourcedMessage +trait MasterSourcedMessage extends SourcedMessage + +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -47,6 +60,9 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { @@ -55,6 +71,8 @@ object Acquire acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { @@ -64,15 +82,15 @@ object Acquire acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) acq } } -class AcquireData extends MemData +class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends PhysicalAddress { +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } object Release @@ -86,35 +104,105 @@ object Release rel } } -class Release extends PhysicalAddress { +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ReleaseData extends MemData +class ReleaseData extends ClientSourcedMessage with HasMemData -class Grant extends MemData { +class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class GrantAck extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) -} +class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } +} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } + +object UncachedTileLinkIOArbiterShim { + def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { + val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} + shim.io.in <> in + shim.io.out + } +} +class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip + val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} + } + io.out.bits := io.in.bits + io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) + io.out.valid := io.in.valid + io.in.ready := io.out.ready +} + + +class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + when(io.out.acquire_data.valid && io.out.acquire_data.ready) { + mem_cnt := mem_cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(mem_cnt_next === UFix(0)) { + locked := Bool(false) + } + } + + val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) + (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) + var acqd_bits = io.in(n-1).acquire_data.bits + for (i <- n-2 to 0 by -1) { + acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) + } + val locked_req = io.in(lock_idx).acquire_data + io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) + io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) + + val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + + val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + val tag = io.out.grant.bits.payload.client_xact_id + io.in(i).grant.valid := Bool(false) + when (tag(log2Up(n)-1,0) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + } +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5609b701..14467f7b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,688 +3,199 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val client_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) } -class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { - val co = conf.co - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nClients) - val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data = (new PipeIO) { new ReleaseData }.flip - val acquire_data = (new PipeIO) { new AcquireData }.flip - val sent_grant_ack = Bool(INPUT) - val release_data_dep = (new PipeIO) { new TrackerDependency }.flip - val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) { new Probe } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val release_client_id = Bits(OUTPUT, conf.ln.idBits) - val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nClients) - val pop_release = Bits(OUTPUT, conf.ln.nClients) - val pop_release_data = Bits(OUTPUT, conf.ln.nClients) - val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) - val send_grant_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << client_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << client_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val xact = Reg{ new Acquire } - val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) - val release_client_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nClients) - probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { - val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) - } - - io.busy := state != s_idle - io.addr := xact.addr - io.init_client_id := init_client_id_ - io.release_client_id := release_client_id_ - io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only - io.a_type := xact.a_type - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.master_xact_id := UFix(id) - io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nClients) - io.pop_release := Bits(0, width = conf.ln.nClients) - io.pop_release_data := Bits(0, width = conf.ln.nClients) - io.pop_release_dep := Bits(0, width = conf.ln.nClients) - io.pop_acquire := Bits(0, width = conf.ln.nClients) - io.pop_acquire_data := Bits(0, width = conf.ln.nClients) - io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) - io.send_grant_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.acquire - init_client_id_ := io.alloc_req.bits.client_id - acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) - x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - probe_flags := probe_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(probe_flags.orR) { - io.push_probe := probe_flags - io.probe.valid := Bool(true) - } - when(io.probe_cnt_inc.orR) { - probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs - } - when(io.release_cnt_dec.orR) { - val dec = PopCount(io.release_cnt_dec) - io.pop_release := io.release_cnt_dec - if(conf.ln.nClients > 1) release_count := release_count - dec - when(release_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - release_data_needs_write := Bool(true) - release_client_id_ := io.p_data.bits.client_id - } - } - is(s_mem) { - when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, - release_data_needs_write, - p_w_mem_cmd_sent, - io.pop_release_data, - io.pop_release_dep, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), - release_client_id_) - } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, - acquire_data_needs_write, - x_w_mem_cmd_sent, - io.pop_acquire_data, - io.pop_acquire_dep, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), - init_client_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_grant_ack := Bool(true) - when(io.sent_grant_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.grant_ack) { - state := s_idle - } - } - } -} - -case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) - -class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val net = (new TileLinkIO).flip - val hub = Vec(conf.nClients) { new TileLinkIO } - } - - val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } - val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - - val hubClientProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - val hubMasterProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - - hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { - net.bits.header.src := UFix(0) - net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} - net.bits.payload := hub(0).bits.payload - net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - hub.foreach( _.ready := net.ready) - }} - hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.bits.header := net.bits.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits.payload := net.bits.payload) - net.ready := hub.map(_.ready).fold(Bool(false))(_||_) - }} -} - -abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val tiles = Vec(conf.nClients) { new TileLinkIO }.flip - val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem - } -} - -class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - val co = conf.co.asInstanceOf[ThreeStateIncoherence] - - val acquire = io.tiles(0).acquire - val is_write = acquire.bits.payload.a_type === co.acquireWriteback - acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := acquire.bits.payload.addr - io.mem.req_data <> io.tiles(0).acquire_data - - val grant = io.tiles(0).grant - grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) - grant.bits.payload.master_xact_id := UFix(0) // don't care - grant.bits.payload.data := io.mem.resp.bits.data - grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).grant_ack.ready := Bool(true) - io.tiles(0).probe.valid := Bool(false) - io.tiles(0).release.ready := Bool(true) - io.tiles(0).release_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - implicit val lnConf = conf.ln - val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } - val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_client_id_arr(i) := t.init_client_id - client_xact_id_arr(i) := t.client_xact_id - a_type_arr(i) := t.a_type - sh_count_arr(i) := t.sharer_count - send_grant_ack_arr(i) := t.send_grant_ack - t.grant_ack := do_free_arr(i) - t.p_data.bits.client_id := p_data_client_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.release_cnt_dec := release_cnt_dec_arr(i).toBits - t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits - t.tile_incoherent := io.incoherent.toBits - t.sent_grant_ack := sent_grant_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_grant_ack_arr(i) := Bool(false) - p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nClients) { - release_cnt_dec_arr(i)(j) := Bool(false) - probe_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until conf.ln.nClients ) { - val ack = io.tiles(j).grant_ack - when (ack.valid) { - do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) - } - ack.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nClients ) { - val rep = io.tiles(j).grant - rep.bits.payload.g_type := UFix(0) - rep.bits.payload.client_xact_id := UFix(0) - rep.bits.payload.master_xact_id := UFix(0) - rep.bits.payload.data := io.mem.resp.bits.data - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) - rep.bits.payload.master_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) - rep.bits.payload.master_xact_id := ack_idx - when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_grant_ack_arr.toBits.orR - sent_grant_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nClients ) { - val release = io.tiles(j).release - val release_data = io.tiles(j).release_data - val idx = release.bits.payload.master_xact_id - val pop_releases = trackerList.map(_.io.pop_release(j).toBool) - val do_pop = foldR(pop_releases)(_ || _) - release.ready := Bool(true) - release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id - release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) - when (release.valid && co.messageHasData(release.bits.payload)) { - p_data_valid_arr(idx) := Bool(true) - p_data_client_id_arr(idx) := UFix(j) - } - release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid - trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until conf.ln.nClients) { - val release = io.tiles(j).release - release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until conf.ln.nClients ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val acquire_data_dep = acquire_data_dep_list(j).io.deq - init_arb.io.in(j).valid := acquire.valid - init_arb.io.in(j).bits.acquire := acquire.bits.payload - init_arb.io.in(j).bits.client_id := UFix(j) - val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - acquire.ready := foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) - acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nClients ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb_arr(j).io.in(i).bits := t.probe.bits - probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) - probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready - } - FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe - } - -} +case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val network = (new TileLinkIO).flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem } } -class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) + val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i) - t.io.tile_incoherent := io.incoherent.toBits - t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) - t.io.mem_resp.bits := io.mem.resp.bits - } - io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val acquire = io.network.acquire - val acquire_data = io.network.acquire_data - val x_dep_deq = acquire_data_dep_q.io.deq + val acquire = io.client.acquire + val acquire_data = io.client.acquire_data + val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } - for( i <- 0 to NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + for( i <- 0 until trackerList.size ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + val t = trackerList(i).io.client + t.acquire.bits := acquire.bits + t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.acquire_data.bits := acquire_data.bits - trackerList(i).io.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid + t.acquire_data.bits := acquire_data.bits + t.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) - x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb.io.in(i).bits := t.probe.bits - probe_arb.io.in(i).valid := t.probe.valid - t.probe.ready := probe_arb.io.in(i).ready - } - io.network.probe <> probe_arb.io.out + val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } // Handle probe replies, which may or may not have data - val release = io.network.release - val release_data = io.network.release_data + val release = io.client.release + val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := release_data.valid - trackerList(i).io.release_data.bits := release_data.bits + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.valid := release.valid && (idx === UFix(i)) + t.release.bits := release.bits + t.release_data.valid := release_data.valid + t.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) - trackerList(i).io.release.bits := release.bits } // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - grant_arb.io.in(i).bits := t.grant.bits - grant_arb.io.in(i).valid := t.grant.valid - t.grant.ready := grant_arb.io.in(i).ready - } - io.network.grant.valid := grant_arb.io.out.valid - io.network.grant.bits := grant_arb.io.out.bits - grant_arb.io.out.ready := io.network.grant.ready - when(io.mem.resp.valid) { - io.network.grant.valid := Bool(true) - io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready - } - } + val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.network.grant_ack - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) - } + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } - for( i <- 0 to NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out } -abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { +abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip - val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val mem_resp = (new FIFOIO) { new MemResp }.flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} - val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} - val busy = Bool(OUTPUT) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip + + val busy = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.mem_resp.ready := Bool(false) - io.probe.valid := Bool(false) + io.release_data_dep.ready := Bool(false) io.busy := Bool(true) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) - io.grant.valid := Bool(false) - io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := cmd_to_write + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header.src := UFix(bankId) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.release.valid ) { - xact := io.release.bits.payload - init_client_id_ := io.release.bits.header.src - release_data_needs_write := co.messageHasData(io.release.bits.payload) + when( io.client.release.valid ) { + xact := io.client.release.bits.payload + init_client_id_ := io.client.release.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.release.ready := Bool(true) + io.client.release.ready := Bool(true) state := s_mem } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . otherwise { state := s_ack } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := s_idle } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } } } } -class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } @@ -697,63 +208,65 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.payload.master_xact_id := UFix(id) - io.probe.bits.payload.addr := xact.addr - io.probe.bits.header.dst := UFix(0) - io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := grant_type - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ - io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header := UFix(bankId) + io.client.probe.valid := Bool(false) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.addr := xact.addr + io.client.probe.bits.header.dst := UFix(0) + io.client.probe.bits.header.src := UFix(bankId) + io.client.grant.bits.payload.data := io.master.grant.bits.payload.data + io.client.grant.bits.payload.g_type := grant_type + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.mem_resp.ready := io.grant.ready + io.master.grant.ready := io.client.grant.ready + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.acquire.valid ) { - xact := io.acquire.bits.payload - init_client_id_ := io.acquire.bits.header.src + when( io.client.acquire.valid ) { + xact := io.client.acquire.bits.payload + init_client_id_ := io.client.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) - x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.acquire.ready := Bool(true) + r_w_mem_cmd_sent := Bool(false) + a_w_mem_cmd_sent := Bool(false) + io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -763,78 +276,76 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends is(s_probe) { val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { - io.probe.valid := Bool(true) - io.probe.bits.header.dst := curr_p_id + io.client.probe.valid := Bool(true) + io.client.probe.bits.header.dst := curr_p_id } - when(io.probe.ready) { + when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.release.valid) { - io.release.ready := Bool(true) + when(io.client.release.valid) { + io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.release.bits.payload) + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, - p_w_mem_cmd_sent, + r_w_mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.acquire_data, acquire_data_needs_write, - x_w_mem_cmd_sent, + a_w_mem_cmd_sent, io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) + doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) + Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.free) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { state := s_idle } } } } -trait MemoryRequestGenerator { +abstract trait OuterRequestGenerator { + val cmd_to_write: Acquire + val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits.payload - when(req_cmd.ready && req_cmd.valid) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + master_acq.bits.payload := cmd_to_write + master_acq_data.bits.payload := client_data.bits.payload + when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - data.ready:= Bool(true) - when (data.valid) { + master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (master_acq.ready || cmd_sent) { + master_acq_data.valid := client_data.valid + when(master_acq_data.ready) { + client_data.ready:= Bool(true) + when (client_data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { pop_dep := Bool(true) @@ -846,10 +357,10 @@ trait MemoryRequestGenerator { } } - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { + def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { + master_acq.valid := Bool(true) + master_acq.bits.payload := cmd_to_read + when(master_acq.ready) { trigger := Bool(false) } } From 4d007d5c404168104d23473d09615c8aee7930fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 169/374] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 460 ++++++++++++++++++++-------------------- 1 file changed, 234 insertions(+), 226 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 01e9c40c..9d736e8b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -404,31 +404,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -437,33 +437,33 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.bits.header.dst := UFix(0) // DNC x_abort.bits.header.src := UFix(0) // DNC - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -472,7 +472,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -490,41 +490,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -542,8 +542,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -556,50 +556,50 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.dst := acquire.bits.header.src x_abort.bits.header.src := UFix(0) //DNC - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -608,38 +608,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -664,11 +664,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -688,20 +688,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -713,32 +713,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -746,12 +746,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) +<<<<<<< HEAD io.p_req.valid := Bool(false) io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) io.p_req.bits.header.src := UFix(0) // DNC +======= + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) +>>>>>>> changed val names in hub to match new tilelink names io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -760,70 +768,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.header.dst := init_client_id_ io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 67fc09f62ec7c5487d33fd851ef4c96477add9d3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 25 Mar 2013 16:20:12 -0700 Subject: [PATCH 170/374] Fixes after merge, and always self probe. --- uncore/src/coherence.scala | 10 ++++---- uncore/src/uncore.scala | 52 ++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b4c4067f..1fa09fe6 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -274,7 +274,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -431,7 +431,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -592,7 +592,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -758,7 +758,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -940,7 +940,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 14467f7b..ea3c20c4 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -37,33 +37,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data + // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + t.acquire.valid := alloc_arb.io.in(i).ready t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -75,22 +73,23 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.valid := release.valid && (idx === UFix(i)) t.release.bits := release.bits - t.release_data.valid := release_data.valid + t.release.valid := release.valid && (idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -121,7 +120,6 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val busy = Bool(OUTPUT) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -139,7 +137,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.acquire_data_dep.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.busy := Bool(true) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -156,7 +153,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.acquire_data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.release_data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -167,13 +164,13 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu switch (state) { is(s_idle) { + io.client.release.ready := Bool(true) when( io.client.release.valid ) { xact := io.client.release.bits.payload init_client_id_ := io.client.release.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.client.release.ready := Bool(true) state := s_mem } } @@ -205,6 +202,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) @@ -216,12 +214,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.busy := state != s_idle io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) io.master.acquire.valid := Bool(false) @@ -256,6 +253,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura switch (state) { is(s_idle) { + io.client.acquire.ready := Bool(true) when( io.client.acquire.valid ) { xact := io.client.acquire.bits.payload init_client_id_ := io.client.acquire.bits.header.src @@ -266,7 +264,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -274,7 +271,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } } is(s_probe) { - val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { io.client.probe.valid := Bool(true) io.client.probe.bits.header.dst := curr_p_id @@ -282,13 +278,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } + io.client.release.ready := Bool(true) when(io.client.release.valid) { - io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { From b6cc08e8ca563617607c07e37c5427f2d2813fef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Mar 2013 14:09:08 -0700 Subject: [PATCH 171/374] override io in LogicalNetwork --- uncore/src/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 492079cc..f353a524 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -49,7 +49,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] + override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } From 3479f1c6cd86bc0d555b1cb0c20660a274a098c6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 3 Apr 2013 22:13:51 -0700 Subject: [PATCH 172/374] add LR/SC support --- uncore/src/coherence.scala | 12 ++++-------- uncore/src/consts.scala | 40 +++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 1fa09fe6..d97cc2da 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -121,8 +121,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) + Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -517,8 +516,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -679,8 +677,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -857,8 +854,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index d348e0ff..cde8d3b9 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -45,27 +45,31 @@ trait MemoryOpConstants { val MT_HU = Bits("b101", 3); val MT_WU = Bits("b110", 3); - val M_X = Bits("b????", 4); - val M_XRD = Bits("b0000", 4); // int load - val M_XWR = Bits("b0001", 4); // int store - val M_PFR = Bits("b0010", 4); // prefetch with intent to read - val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FENCE = Bits("b0101", 4); // memory fence - val M_INV = Bits("b0110", 4); // write back and invalidate line - val M_CLN = Bits("b0111", 4); // write back line - val M_XA_ADD = Bits("b1000", 4); - val M_XA_SWAP = Bits("b1001", 4); - val M_XA_AND = Bits("b1010", 4); - val M_XA_OR = Bits("b1011", 4); - val M_XA_MIN = Bits("b1100", 4); - val M_XA_MAX = Bits("b1101", 4); - val M_XA_MINU = Bits("b1110", 4); - val M_XA_MAXU = Bits("b1111", 4); + val M_SZ = 5 + val M_X = Bits("b?????"); + val M_XRD = Bits("b00000"); // int load + val M_XWR = Bits("b00001"); // int store + val M_PFR = Bits("b00010"); // prefetch with intent to read + val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_FENCE = Bits("b00101"); // memory fence + val M_XLR = Bits("b00110"); + val M_XSC = Bits("b00111"); + val M_XA_ADD = Bits("b01000"); + val M_XA_SWAP = Bits("b01001"); + val M_XA_AND = Bits("b01010"); + val M_XA_OR = Bits("b01011"); + val M_XA_MIN = Bits("b01100"); + val M_XA_MAX = Bits("b01101"); + val M_XA_MINU = Bits("b01110"); + val M_XA_MAXU = Bits("b01111"); + val M_INV = Bits("b10000"); // write back and invalidate line + val M_CLN = Bits("b10001"); // write back line def isAMO(cmd: Bits) = cmd(3) def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW - def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) - def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) + def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } trait MemoryInterfaceConstants extends From 7ff5b5b86f1dfa508b4c67a3d2c1127da7611f61 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 7 Apr 2013 19:23:44 -0700 Subject: [PATCH 173/374] treat load-reserved as a non-dirtying store --- uncore/src/coherence.scala | 66 +++++++++++++------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d97cc2da..20157456 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -7,10 +7,6 @@ trait CoherenceAgentRole trait ClientCoherenceAgent extends CoherenceAgentRole trait MasterCoherenceAgent extends CoherenceAgentRole -object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) -} - abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -102,8 +98,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def needsWriteback (state: UFix): Bool = state === tileDirty def newState(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) + Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO @@ -124,8 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData @@ -296,9 +290,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusiveDirty)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -311,8 +304,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -352,12 +344,10 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -450,8 +440,7 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -459,9 +448,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -474,8 +462,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -519,8 +506,7 @@ class MSICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -611,8 +597,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -620,9 +605,8 @@ class MESICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -635,8 +619,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -680,8 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -775,17 +757,15 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UFix): Bool = { state != tileInvalid } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -798,8 +778,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } @@ -857,8 +836,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) From 74187c2068144b640347fa674014bfb868e3428e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Apr 2013 14:09:36 -0700 Subject: [PATCH 174/374] Always route voluntary releases to ReleaseTracker to ensure all grants are sent --- uncore/src/uncore.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ea3c20c4..3e54b9e3 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -68,27 +68,28 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } - // Handle probe replies, which may or may not have data + // Handle releases, which might be voluntary and might have data val release = io.client.release val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits - t.release.valid := release.valid && (idx === UFix(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.enq.bits.master_xact_id := release_idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor From 766d5622b17d432286a11c1a98ee46423ad55912 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 10 Apr 2013 13:46:31 -0700 Subject: [PATCH 175/374] Prevent messages from becoming interleaved in the BasicCrossbar. Remove dependency trackers from the uncore, use msg headers instead. Have one ReleaseHnadler per core for now. --- uncore/src/coherence.scala | 10 ++--- uncore/src/package.scala | 53 +++++++++++++++++++++++- uncore/src/tilelink.scala | 5 ++- uncore/src/uncore.scala | 82 +++++++++++++------------------------- 4 files changed, 86 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 20157456..ec932ec9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -265,7 +265,7 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) @@ -418,7 +418,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -574,7 +574,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -735,7 +735,7 @@ class MESICoherence extends CoherencePolicyWithUncached { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -912,7 +912,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index f353a524..4133a9d3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -24,7 +24,56 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { + val io = new ioArbiter(n)(data) + + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + if(count > 1){ + when(io.out.valid && io.out.ready) { + cnt := cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + } else { + locked := Bool(false) + lock_idx := UFix(n) + cnt := UFix(0) + } + + + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) + val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) + (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(locked, lock_idx, choose) + when (io.out.valid && io.out.ready) { + last_grant := choose + } + + val dvec = Vec(n) { data } + (0 until n).map(i => dvec(i) := io.in(i).bits ) + + io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) + io.out.bits := dvec(choose) + io.chosen := choose +} + +class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} @@ -33,7 +82,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16f8c9b..c7f82da2 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -118,11 +118,12 @@ class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -131,7 +132,7 @@ class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bun class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3e54b9e3..445444db 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,18 +3,10 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val client_id = Bits(width = conf.ln.idBits) -} - -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { - val acquire = new Acquire() - val client_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } -} - -class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { + val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) + val data_src_id = Bits(width = conf.ln.idBits) + override def clone = { new TrackerDependency().asInstanceOf[this.type] } } case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) @@ -31,18 +23,16 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) - val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data - val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) + val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { @@ -53,14 +43,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) - acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation @@ -73,24 +58,19 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) + val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid - trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release_idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -116,17 +96,13 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } @@ -136,8 +112,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire_data_dep.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -159,6 +133,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.data := UFix(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant_ack.valid := Bool(false) @@ -181,9 +156,8 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.master.acquire_data, io.client.release_data, release_data_needs_write, - mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + mem_cmd_sent, + init_client_id_) } . otherwise { state := s_ack } } is(s_ack) { @@ -198,6 +172,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) @@ -245,10 +220,8 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.ready := Bool(false) io.client.acquire_data.ready := Bool(false) - io.acquire_data_dep.ready := Bool(false) io.client.release.ready := Bool(false) io.client.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) @@ -285,7 +258,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) + when( co.messageHasData(io.client.release.bits.payload)) { + release_data_needs_write := Bool(true) + release_data_client_id := io.client.release.bits.header.src + } } } is(s_mem) { @@ -295,16 +271,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.release_data, release_data_needs_write, r_w_mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.master.acquire_data, io.client.acquire_data, acquire_data_needs_write, a_w_mem_cmd_sent, - io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) + init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { @@ -330,24 +304,22 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.bits.payload := cmd_to_write master_acq_data.bits.payload := client_data.bits.payload when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } - when (at_front_of_dep_queue) { - master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (do_write) { + master_acq.valid := !cmd_sent && master_acq_data.ready when (master_acq.ready || cmd_sent) { master_acq_data.valid := client_data.valid when(master_acq_data.ready) { client_data.ready:= Bool(true) - when (client_data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := Bool(true) - trigger := Bool(false) - } + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + trigger := Bool(false) } } } From 12d394811efccb7e1f83b3935695235b24be447d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Apr 2013 18:47:37 -0700 Subject: [PATCH 176/374] Allow release data to be written out even before all releases have been collected --- uncore/src/uncore.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 445444db..5dcb192f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -263,6 +263,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura release_data_client_id := io.client.release.bits.header.src } } + when (release_data_needs_write) { + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, + release_data_needs_write, + r_w_mem_cmd_sent, + release_data_client_id) + } } is(s_mem) { when (release_data_needs_write) { From fedc2753e4c54476f34c4ff1b559c1b8baf40b46 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Apr 2013 11:00:23 -0700 Subject: [PATCH 177/374] make sure master_xact_id field is large enough for temporary extra release trackers --- uncore/src/consts.scala | 5 +++-- uncore/src/uncore.scala | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index cde8d3b9..9d343b3f 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,8 +10,9 @@ abstract trait CoherenceConfigConstants { } trait UncoreConstants { - val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) + val NGLOBAL_ACQ_XACTS = 8 + val NGLOBAL_REL_XACTS = 4 + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5dcb192f..f0571d89 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,8 +23,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) + require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) From 9a258e7fb467e5d3135fc7b372ef73cb9428ede5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 178/374] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From 9a3b2e7006c370b6c37bee57272a0d1788d50e5b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 1 May 2013 16:48:01 -0700 Subject: [PATCH 179/374] new paired meta/data IO type, and matching arbiter --- uncore/src/package.scala | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7e1fc7a0..46c39d51 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -8,6 +8,68 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = (new FIFOIO()){ m } + val data = (new FIFOIO()){ d } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { From 0672773c1a762db43ad66cf6e3e70f2bff6e061f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 9 May 2013 02:14:44 -0700 Subject: [PATCH 180/374] for now, don't use asserts outside of components --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..42b3495e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -842,7 +842,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), From 4c1f105ce94eb9dd3ae7902272f5301119278c15 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 May 2013 17:19:07 -0700 Subject: [PATCH 181/374] added PairedData link type with matching crossbar, ported tilelink and uncore to use --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 50 +++++++++-- uncore/src/tilelink.scala | 93 ++++++++------------ uncore/src/uncore.scala | 168 ++++++++++++++++++------------------- 4 files changed, 158 insertions(+), 159 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..6a1eaf80 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgentRole -trait ClientCoherenceAgent extends CoherenceAgentRole -trait MasterCoherenceAgent extends CoherenceAgentRole +abstract trait CoherenceAgentRole +abstract trait ClientCoherenceAgent extends CoherenceAgentRole +abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46c39d51..754621ad 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -9,8 +9,9 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = (new FIFOIO()){ m } - val data = (new FIFOIO()){ d } + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { @@ -18,9 +19,10 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) @@ -28,11 +30,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) - val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) io.meta_chosen := meta_chosen @@ -70,6 +74,34 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -88,7 +120,7 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c7f82da2..589254f9 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -116,82 +116,55 @@ class Grant extends MasterSourcedMessage with HasMemData with HasClientTransacti class GrantAck extends ClientSourcedMessage with HasMasterTransactionId -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} +trait DirectionalIO +trait ClientSourcedIO extends DirectionalIO +trait MasterSourcedIO extends DirectionalIO +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} - val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) + val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) + val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} + val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) + val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -object UncachedTileLinkIOArbiterShim { - def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { - val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} - shim.io.in <> in - shim.io.out - } -} -class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip - val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} - } - io.out.bits := io.in.bits - io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) - io.out.valid := io.in.valid - io.in.ready := io.out.ready -} - - -class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - when(io.out.acquire_data.valid && io.out.acquire_data.ready) { - mem_cnt := mem_cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(mem_cnt_next === UFix(0)) { - locked := Bool(false) - } - } - - val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) - (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) - var acqd_bits = io.in(n-1).acquire_data.bits - for (i <- n-2 to 0 by -1) { - acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) - } - val locked_req = io.in(lock_idx).acquire_data - io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) - io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) - - val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + req.meta.ready := arb.meta.ready + }} - val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index f0571d89..3a3c36ae 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -31,23 +31,22 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle acquire transaction initiation val acquire = io.client.acquire - val acquire_data = io.client.acquire_data val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + alloc_arb.io.in(i).valid := t.acquire.meta.ready + t.acquire.meta.bits := acquire.meta.bits + t.acquire.meta.valid := alloc_arb.io.in(i).ready - t.acquire_data.bits := acquire_data.bits - t.acquire_data.valid := acquire_data.valid + t.acquire.data.bits := acquire.data.bits + t.acquire.data.valid := acquire.data.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires - acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires + acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) + alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -56,22 +55,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle releases, which might be voluntary and might have data val release = io.client.release - val release_data = io.client.release_data - val voluntary = co.isVoluntary(release.bits.payload) + val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases - t.release_data.bits := release_data.bits - t.release_data.valid := release_data.valid + t.release.meta.bits := release.meta.bits + t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.data.bits := release.data.bits + t.release.data.valid := release.data.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -85,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } @@ -114,22 +112,22 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := cmd_to_write + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header.src := UFix(bankId) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) + io.master.acquire.data.bits.header.src := UFix(bankId) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) // DNC + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -141,11 +139,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { - xact := io.client.release.bits.payload - init_client_id_ := io.client.release.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + io.client.release.meta.ready := Bool(true) + when( io.client.release.meta.valid ) { + xact := io.client.release.meta.bits.payload + init_client_id_ := io.client.release.meta.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) state := s_mem @@ -153,10 +151,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, mem_cmd_sent, init_client_id_) } . otherwise { state := s_ack } @@ -192,20 +189,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header := UFix(bankId) + io.master.acquire.data.bits.header := UFix(bankId) io.client.probe.valid := Bool(false) io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.client.probe.bits.payload.master_xact_id := UFix(trackerId) @@ -219,22 +216,22 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) - when( io.client.acquire.valid ) { - xact := io.client.acquire.bits.payload - init_client_id_ := io.client.acquire.bits.header.src + io.client.acquire.meta.ready := Bool(true) + when( io.client.acquire.meta.valid ) { + xact := io.client.acquire.meta.bits.payload + init_client_id_ := io.client.acquire.meta.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) @@ -253,21 +250,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - io.client.release.ready := Bool(true) - when(io.client.release.valid) { + io.client.release.meta.ready := Bool(true) + when(io.client.release.meta.valid) { if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - when( co.messageHasData(io.client.release.bits.payload)) { + when( co.messageHasData(io.client.release.meta.bits.payload)) { release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.bits.header.src + release_data_client_id := io.client.release.meta.bits.header.src } } when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, + doOuterReqWrite(io.master.acquire, + io.client.release.data, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -275,18 +271,16 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, - r_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, + r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.acquire_data, - acquire_data_needs_write, - a_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.acquire.data, + acquire_data_needs_write, + a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) @@ -313,18 +307,18 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.bits.payload := cmd_to_write - master_acq_data.bits.payload := client_data.bits.payload - when(master_acq.ready && master_acq.valid) { + master_acq.meta.bits.payload := cmd_to_write + master_acq.data.bits.payload := client_data.bits.payload + when(master_acq.meta.fire()) { cmd_sent := Bool(true) } when (do_write) { - master_acq.valid := !cmd_sent && master_acq_data.ready - when (master_acq.ready || cmd_sent) { - master_acq_data.valid := client_data.valid - when(master_acq_data.ready) { + master_acq.meta.valid := !cmd_sent + when (master_acq.meta.ready || cmd_sent) { + master_acq.data.valid := client_data.valid + when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { @@ -335,10 +329,10 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { - master_acq.valid := Bool(true) - master_acq.bits.payload := cmd_to_read - when(master_acq.ready) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + master_acq.meta.valid := Bool(true) + master_acq.meta.bits.payload := cmd_to_read + when(master_acq.meta.ready) { trigger := Bool(false) } } From cf02f1ef014ab7f644f1657ecd97f37152cda36f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 182/374] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From db8e5fda9b908e908f78110012836dbab5b80c72 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Jul 2013 15:37:42 -0700 Subject: [PATCH 183/374] new tilelink arbiter types, reduced release xact trackers --- uncore/src/consts.scala | 2 +- uncore/src/tilelink.scala | 96 ++++++++++++++++++++++++++++++++++++--- uncore/src/uncore.scala | 6 +-- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 9d343b3f..aaa7b7a8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 4 + val NGLOBAL_REL_XACTS = 1 val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 589254f9..2e595c34 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -147,12 +147,24 @@ class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTil override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +/* + * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int): Bits + def grantClientXactId(in: Grant): Bits + def arbIdx(in: Grant): UFix +} +*/ + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out @@ -160,7 +172,7 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi arb.data <> req.data arb.meta.valid := req.meta.valid arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) req.meta.ready := arb.meta.ready }} @@ -170,13 +182,85 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi io.out.grant.ready := Bool(false) for (i <- 0 until n) { - val tag = io.out.grant.bits.payload.client_xact_id io.in(i).grant.valid := Bool(false) - when (tag(log2Up(n)-1,0) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) } } + +class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id + def grantClientXactId(in: Grant) = in.client_xact_id + def arbIdx(in: Grant): UFix = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) + def grantClientXactId(in: Grant) = UFix(0) // DNC + def arbIdx(in: Grant) = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3a3c36ae..9d0de31b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,7 +23,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) @@ -60,7 +60,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits @@ -83,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } From d8440b042a4d74c3f5f437ab73b328c5ff1ad94c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Jul 2013 23:22:36 -0700 Subject: [PATCH 184/374] Make compatible with scala 2.10. Refactor constants into package object. Remove networking primitives from package object. Clean up request generators. Chnage ++ to +: for appending to io.incoherent. --- uncore/src/coherence.scala | 2 - uncore/src/consts.scala | 2 + uncore/src/llc.scala | 3 - uncore/src/memserdes.scala | 3 - uncore/src/network.scala | 193 ++++++++++++++++++++++++++++++++++ uncore/src/package.scala | 210 ++----------------------------------- uncore/src/slowio.scala | 2 - uncore/src/temp.scala | 13 --- uncore/src/tilelink.scala | 2 - uncore/src/uncore.scala | 19 ++-- 10 files changed, 211 insertions(+), 238 deletions(-) create mode 100644 uncore/src/network.scala delete mode 100644 uncore/src/temp.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 03c5c8f1..b869436a 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ abstract trait CoherenceAgentRole abstract trait ClientCoherenceAgent extends CoherenceAgentRole diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index aaa7b7a8..4e3c814e 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -36,6 +36,7 @@ trait TileLinkSizeConstants extends val ACQUIRE_ATOMIC_OP_BITS = 4 } +object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); val MT_B = Bits("b000", 3); @@ -82,6 +83,7 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } +object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 val VADDR_BITS = 43; diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 1f9f4b2a..6b0c77f2 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 88a6f1b3..c7c8e773 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ import scala.math._ class ioMemSerialized(w: Int) extends Bundle diff --git a/uncore/src/network.scala b/uncore/src/network.scala new file mode 100644 index 00000000..cdaa8118 --- /dev/null +++ b/uncore/src/network.scala @@ -0,0 +1,193 @@ +package uncore +import Chisel._ +import scala.collection.mutable.Stack + +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data + override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + } + + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { + override val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src + io.in.ready := io.out.ready +} + +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 754621ad..a92f739c 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,204 +1,8 @@ -package object uncore { -import Chisel._ -import Node._ -import scala.collection.mutable.Stack - -//TODO: Remove these Networking classes from the package object once Scala bug -//SI-3439 is resolved. - -implicit def toOption[A](a: A) = Option(a) - -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { - require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} - } - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} - -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } -} - -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} - } - - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} - (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) - arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} - -object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} - -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } -} - +package object uncore extends + uncore.constants.MemoryOpConstants with + uncore.constants.MemoryInterfaceConstants with + uncore.constants.CacheConstants with + uncore.constants.AddressConstants +{ + implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index cf00ae6e..b56b1396 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala deleted file mode 100644 index b2d3fdca..00000000 --- a/uncore/src/temp.scala +++ /dev/null @@ -1,13 +0,0 @@ -package uncore -import _root_.uncore.constants._ - -//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and -// mixin Constants traits to package object uncore in package.scala and -// remove import Constants._'s from other .scala files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ -} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 2e595c34..3621e73b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 9d0de31b..5f87f724 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) @@ -109,7 +107,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -153,6 +150,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, mem_cmd_sent, init_client_id_) @@ -264,6 +262,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -273,17 +272,19 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.acquire.data, + cmd_to_write, acquire_data_needs_write, a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, x_needs_read) + doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) @@ -302,14 +303,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } abstract trait OuterRequestGenerator { - val cmd_to_write: Acquire - val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd_to_write + master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload when(master_acq.meta.fire()) { cmd_sent := Bool(true) @@ -329,9 +328,9 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd_to_read + master_acq.meta.bits.payload := cmd when(master_acq.meta.ready) { trigger := Bool(false) } From bc2b45da129883748d197abab5fd50ec3a885a9a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Aug 2013 14:55:06 -0700 Subject: [PATCH 185/374] Fold uncore constants into TileLinkConfiguration, update coherence API --- uncore/src/coherence.scala | 229 +++++++++++++++++++++---------------- uncore/src/consts.scala | 61 ++++------ uncore/src/llc.scala | 12 +- uncore/src/memserdes.scala | 27 +++++ uncore/src/package.scala | 5 +- uncore/src/tilelink.scala | 122 ++++++++++---------- uncore/src/uncore.scala | 75 ++++++------ 7 files changed, 283 insertions(+), 248 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b869436a..b8c3e4ad 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -6,6 +6,19 @@ abstract trait ClientCoherenceAgent extends CoherenceAgentRole abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { + def nClientStates: Int + def nMasterStates: Int + def nAcquireTypes: Int + def nProbeTypes: Int + def nReleaseTypes: Int + def nGrantTypes: Int + def clientStateBits = log2Up(nClientStates) + def masterStateBits = log2Up(nMasterStates) + def acquireTypeBits = log2Up(nAcquireTypes) + def probeTypeBits = log2Up(nProbeTypes) + def releaseTypeBits = log2Up(nReleaseTypes) + def grantTypeBits = log2Up(nGrantTypes) + def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -22,11 +35,12 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getProbeType(a_type: UFix, global_state: UFix): UFix def getReleaseTypeOnCacheControl(cmd: Bits): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix, id: UFix): Release + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits + def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -35,9 +49,6 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits - def getProbeType(a_type: UFix, global_state: UFix): UFix def needsOuterRead(a_type: UFix, global_state: UFix): Bool def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool @@ -50,11 +61,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire - def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def getUncachedReadAcquireType: Bits + def getUncachedWriteAcquireType: Bits + def getUncachedReadWordAcquireType: Bits + def getUncachedWriteWordAcquireType: Bits + def getUncachedAtomicAcquireType: Bits def isUncachedReadTransaction(acq: Acquire): Bool } @@ -63,7 +74,7 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -79,10 +90,16 @@ abstract class IncoherentPolicy extends CoherencePolicy { } class ThreeStateIncoherence extends IncoherentPolicy { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } + def nClientStates = 3 + def nMasterStates = 0 + def nAcquireTypes = 3 + def nProbeTypes = 0 + def nReleaseTypes = 2 + def nGrantTypes = 3 + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -109,7 +126,6 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -133,14 +149,21 @@ class ThreeStateIncoherence extends IncoherentPolicy { } class MICoherence extends CoherencePolicyWithUncached { + def nClientStates = 2 + def nMasterStates = 2 + def nAcquireTypes = 6 + def nProbeTypes = 2 + def nReleaseTypes = 5 + def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) @@ -186,12 +209,11 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -200,8 +222,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,7 +231,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -270,14 +291,20 @@ class MICoherence extends CoherencePolicyWithUncached { } class MEICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 2 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -331,12 +358,11 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -349,8 +375,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -361,7 +386,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -424,14 +449,21 @@ class MEICoherence extends CoherencePolicyWithUncached { } class MSICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) @@ -490,12 +522,11 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -508,8 +539,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -520,7 +550,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -580,14 +610,20 @@ class MSICoherence extends CoherencePolicyWithUncached { } class MESICoherence extends CoherencePolicyWithUncached { + def nClientStates = 4 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -647,12 +683,11 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -665,8 +700,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -677,7 +711,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -741,13 +775,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } class MigratoryCoherence extends CoherencePolicyWithUncached { + def nClientStates = 7 + def nMasterStates = 0 + def nAcquireTypes = 8 + def nProbeTypes = 4 + def nReleaseTypes = 11 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -820,12 +860,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -838,22 +877,20 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 4e3c814e..15bfee4d 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -4,38 +4,6 @@ package constants import Chisel._ import scala.math.max -abstract trait CoherenceConfigConstants { - val ENABLE_SHARING: Boolean - val ENABLE_CLEAN_EXCLUSIVE: Boolean -} - -trait UncoreConstants { - val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 1 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 -} - -trait CacheConstants extends UncoreConstants { - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkTypeConstants { - val ACQUIRE_TYPE_MAX_BITS = 2 - val GRANT_TYPE_MAX_BITS = 3 - val PROBE_TYPE_MAX_BITS = 2 - val RELEASE_TYPE_MAX_BITS = 3 -} - -trait TileLinkSizeConstants extends - TileLinkTypeConstants -{ - val CLIENT_XACT_ID_MAX_BITS = 10 - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); @@ -74,15 +42,6 @@ trait MemoryOpConstants { def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } -trait MemoryInterfaceConstants extends - UncoreConstants with - TileLinkSizeConstants -{ - val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS -} - object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 @@ -94,3 +53,23 @@ trait AddressConstants { val PERM_BITS = 6; } +trait CacheConstants { + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + +trait TileLinkSizeConstants { + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 +} + +trait MemoryInterfaceConstants extends + CacheConstants with + AddressConstants +{ + val MEM_TAG_BITS = 5 + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS +} diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 6b0c77f2..cb6465c5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -86,10 +86,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val cpu = (new FIFOIO) { new MemReqCmd }.flip val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UFix(width = MEM_ADDR_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMemPipe @@ -105,7 +105,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillCount = UFix(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -184,7 +184,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component class LLCWriteback(requestors: Int) extends Component { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } val mem = new ioMemPipe } @@ -235,7 +235,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } val writeback_data = (new FIFOIO) { new MemData } val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip @@ -348,7 +348,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val mem = new ioMemPipe } - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = (new Arbiter(2)) { new MemReqCmd } diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c7c8e773..b1272b28 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,6 +2,33 @@ package uncore import Chisel._ import scala.math._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends Bundle { + val addr = UFix(width = MEM_ADDR_BITS) + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends Bundle { + val tag = Bits(width = MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + class ioMemSerialized(w: Int) extends Bundle { val req = (new FIFOIO) { Bits(width = w) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a92f739c..4ac83d0b 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,8 +1,7 @@ package object uncore extends uncore.constants.MemoryOpConstants with - uncore.constants.MemoryInterfaceConstants with - uncore.constants.CacheConstants with - uncore.constants.AddressConstants + uncore.constants.TileLinkSizeConstants with + uncore.constants.MemoryInterfaceConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 3621e73b..6f7e6175 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,99 +1,86 @@ package uncore import Chisel._ -trait HasPhysicalAddress extends Bundle { +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) + +abstract trait TileLinkSubBundle extends Bundle { + implicit val conf: TileLinkConfiguration +} + +trait HasPhysicalAddress extends TileLinkSubBundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +trait HasClientTransactionId extends TileLinkSubBundle { + val client_xact_id = Bits(width = conf.clientXactIdBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +trait HasMasterTransactionId extends TileLinkSubBundle { + val master_xact_id = Bits(width = conf.masterXactIdBits) } -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemData extends Bundle with HasMemData - -class MemReqCmd extends Bundle with HasPhysicalAddress { - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp extends Bundle with HasMemData { - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip +trait HasTileLinkData extends TileLinkSubBundle { + val data = Bits(width = conf.dataBits) } trait SourcedMessage extends Bundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) -} - object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.write_mask := Bits(0) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.write_mask := Bits(0) acq } } +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + val a_type = Bits(width = conf.co.acquireTypeBits) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + override def clone = { (new Acquire).asInstanceOf[this.type] } +} -class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = PROBE_TYPE_MAX_BITS) +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new AcquireData).asInstanceOf[this.type] } +} + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + val p_type = Bits(width = conf.co.probeTypeBits) + override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -102,17 +89,24 @@ object Release rel } } -class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + val r_type = Bits(width = conf.co.releaseTypeBits) + override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData extends ClientSourcedMessage with HasMemData - -class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = GRANT_TYPE_MAX_BITS) +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new ReleaseData).asInstanceOf[this.type] } +} + +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { + val g_type = Bits(width = conf.co.grantTypeBits) + override def clone = { (new Grant).asInstanceOf[this.type] } +} + +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { + override def clone = { (new GrantAck).asInstanceOf[this.type] } } -class GrantAck extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO @@ -132,14 +126,15 @@ class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) exten override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { + implicit val ln = conf.ln val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } @@ -154,7 +149,8 @@ abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit c } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix @@ -190,7 +186,8 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy) } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UFix = in.client_xact_id @@ -226,7 +223,8 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implici } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) def grantClientXactId(in: Grant) = UFix(0) // DNC def arbIdx(in: Grant) = in.client_xact_id diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5f87f724..60e100e0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,30 +1,26 @@ package uncore import Chisel._ -class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { - val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) - val data_src_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerDependency().asInstanceOf[this.type] } -} - -case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) - -abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.nClients) { Bool() }.asInput + val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput } } -class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) +case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) + +class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) { - implicit val lnConf = conf.ln - val co = conf.co - //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config - val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation @@ -81,32 +77,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { - val co = conf.co - implicit val ln = conf.ln +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -163,38 +158,38 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nClients) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { + if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst io.master.acquire.meta.bits.header.src := UFix(bankId) io.master.acquire.data.valid := Bool(false) @@ -227,14 +222,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - if(conf.ln.nClients > 1) { + if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -250,7 +245,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(conf.ln.nClients > 1) release_count := release_count - UFix(1) + if(ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } @@ -306,7 +301,7 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload From 5c7a1f5cd6b4331807e5899f30e1a9348cc8d428 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 10:36:44 -0700 Subject: [PATCH 186/374] initial attempt at upgrade --- uncore/src/coherence.scala | 426 ++++++++++++++++++------------------- uncore/src/llc.scala | 234 ++++++++++---------- uncore/src/memserdes.scala | 129 +++++------ uncore/src/network.scala | 92 ++++---- uncore/src/slowio.scala | 61 +++--- uncore/src/tilelink.scala | 69 +++--- uncore/src/uncore.scala | 134 ++++++------ 7 files changed, 574 insertions(+), 571 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b8c3e4ad..7109be94 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -12,35 +12,35 @@ abstract class CoherencePolicy { def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int - def clientStateBits = log2Up(nClientStates) - def masterStateBits = log2Up(nMasterStates) - def acquireTypeBits = log2Up(nAcquireTypes) - def probeTypeBits = log2Up(nProbeTypes) - def releaseTypeBits = log2Up(nReleaseTypes) - def grantTypeBits = log2Up(nGrantTypes) + def clientStateWidth = log2Up(nClientStates) + def masterStateWidth = log2Up(nMasterStates) + def acquireTypeWidth = log2Up(nAcquireTypes) + def probeTypeWidth = log2Up(nProbeTypes) + def releaseTypeWidth = log2Up(nReleaseTypes) + def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: Bits, state: UFix): Bool - def isValid (state: UFix): Bool + def isHit (cmd: UInt, state: UInt): Bool + def isValid (state: UInt): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool - def needsWriteback (state: UFix): Bool + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool + def needsWriteback (state: UInt): Bool - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnCacheControl(cmd: Bits): UFix - def newStateOnWriteback(): UFix - def newStateOnFlush(): UFix - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix - def newStateOnProbe(incoming: Probe, state: UFix): Bits + def newStateOnHit(cmd: UInt, state: UInt): UInt + def newStateOnCacheControl(cmd: UInt): UInt + def newStateOnWriteback(): UInt + def newStateOnFlush(): UInt + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt + def newStateOnProbe(incoming: Probe, state: UInt): Bits - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getProbeType(a_type: UFix, global_state: UFix): UFix - def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, global_state: UInt): UInt + def getReleaseTypeOnCacheControl(cmd: UInt): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits + def getGrantType(a_type: UInt, count: UInt): Bits + def getGrantType(rel: Release, count: UInt): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -49,15 +49,15 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UFix, global_state: UFix): Bool - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool - def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UInt, global_state: UInt): Bool + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool + def needsAckReply(a_type: UInt, global_state: UInt): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -73,19 +73,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): Bits = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) - def getGrantType(rel: Release, count: UFix): Bits = Bits(0) - def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) + def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -96,27 +96,27 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) val hasDataGrantTypeList = List(grantData) - def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UFix): Bool = state != tileInvalid + def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = state === tileDirty + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty + def needsWriteback (state: UInt): Bool = state === tileDirty - def newState(cmd: Bits, state: UFix): UFix = { + def newState(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) + def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { @@ -129,23 +129,23 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -156,35 +156,35 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), M_CLN -> (state === tileValid) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = state - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = state + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileValid @@ -192,7 +192,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> tileValid, grantReadUncached -> tileInvalid, @@ -202,7 +202,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -218,11 +218,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +235,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,13 +258,13 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -275,19 +275,19 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -298,40 +298,40 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileExclusiveClean @@ -339,7 +339,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), grantReadExclusiveAck -> tileExclusiveDirty, @@ -350,7 +350,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -367,15 +367,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +390,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,14 +413,14 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -432,20 +432,20 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -456,45 +456,45 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -502,7 +502,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> tileExclusiveDirty, @@ -514,7 +514,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -531,15 +531,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,21 +554,21 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -577,14 +577,14 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -593,20 +593,20 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -617,45 +617,45 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -663,7 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), @@ -675,7 +675,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -692,15 +692,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,21 +715,21 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -738,14 +738,14 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -757,13 +757,13 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } @@ -771,7 +771,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -782,45 +782,45 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: Bits, state: UFix): Bool = { - Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + def isHit (cmd: UInt, state: UInt): Bool = { + Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -828,7 +828,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( @@ -846,7 +846,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -869,17 +869,17 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,21 +894,21 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -918,14 +918,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -938,18 +938,18 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb6465c5..3e7c2cd5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,17 +1,17 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module { class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) + val addr = UInt(INPUT, log2Up(n)) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput override def clone = new Inputs().asInstanceOf[this.type] } val io = new Bundle { - val in = new PipeIO()(new Inputs).flip + val in = Valid(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,27 +21,27 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } + val reg_ren = RegUpdate(ren) + val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val ridx = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg(Bits()) else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -74,47 +74,49 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit class LLCDataReq(ways: Int) extends MemReqCmd { - val way = UFix(width = log2Up(ways)) + val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +class LLCTagReq(ways: Int) extends HasMemAddr +{ + val way = UInt(width = log2Up(ways)) + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module { val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) + val cpu = Decoupled(new MemReqCmd).flip + val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = MEM_ADDR_BITS) - val way = UFix(width = log2Up(ways)) - } } + val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val data = Decoupled(new LLCDataReq(ways)) + val tag = Decoupled(new LLCTagReq(ways)) val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) + val mem_resp_set = UInt(OUTPUT, log2Up(sets)) + val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val valid = Vec.fill(outstanding){RegReset(Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } + val mshr = Vec.fill(outstanding){Reg(new MSHR)} when (io.cpu.valid && io.cpu.ready) { valid(freeId) := Bool(true) mshr(freeId).addr := io.cpu.bits.addr @@ -124,7 +126,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).old_tag := io.repl_tag mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) + mshr(freeId).refillCount := UInt(0) mshr(freeId).refilled := Bool(false) } @@ -136,8 +138,8 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + mshr(refillId).refillCount := refillCount + UInt(1) + when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -161,7 +163,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt io.data.bits.way := mshr(writebackId).way when (replay) { io.data.valid := io.tag.ready @@ -181,25 +183,25 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Component +class LLCWriteback(requestors: Int) extends Module { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new ioMemPipe } - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val valid = RegReset(Bool(false)) + val who = Reg(UInt()) + val addr = Reg(UInt()) + val cmd_sent = Reg(Bool()) + val data_sent = Reg(Bool()) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready anyReq = anyReq || io.req(i).valid } @@ -213,8 +215,8 @@ class LLCWriteback(requestors: Int) extends Component } when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { + count := count + UInt(1) + when (count === UInt(REFILL_CYCLES-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -230,51 +232,51 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) + val req = Decoupled(new LLCDataReq(ways)).flip + val req_data = Decoupled(new MemData).flip + val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp) + val mem_resp = Valid(new MemResp).flip + val mem_resp_set = UInt(INPUT, log2Up(sets)) + val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) + val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val q = Module(new Queue(new QEntry, latency+2)) + val qReady = q.io.count <= UInt(q.entries-latency-1) + val valid = RegReset(Bool(false)) + val req = Reg(io.req.bits.clone) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + count := count + UInt(1) + when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt data.io.in.bits.rw := Bool(true) data.io.in.bits.wdata := io.mem_resp.bits.data } @@ -298,21 +300,21 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Component // UNTESTED +class MemReqArb(n: Int) extends Module // UNTESTED { val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } + val cpu = Vec.fill(n){new ioMem().flip} val mem = new ioMem } - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } + val lock = RegReset(Bool(false)) + val locker = Reg(UInt()) - val arb = new RRArbiter(n)(new MemReqCmd) + val arb = Module(new RRArbiter(new MemReqCmd, n)) val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) + val me = UInt(i, log2Up(n)) arb.io.in(i).valid := io.cpu(i).req_cmd.valid arb.io.in(i).bits := io.cpu(i).req_cmd.bits arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) @@ -323,7 +325,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED val haveLock = lock && locker === me when (getLock) { lock := Bool(true) - locker := UFix(i) + locker := UInt(i) } when (getLock || haveLock) { io.cpu(i).req_data.ready := io.mem.req_data.ready @@ -341,7 +343,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -351,48 +353,48 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) + val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) + val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) + val data = Module(new LLCData(4, sets, ways, dataLeaf)) + val writeback = Module(new LLCWriteback(2)) - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initCount = RegReset(UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } + when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = Reg(resetVal = Bool(false)) - val s2_valid = Reg(resetVal = Bool(false)) - val s2 = Reg() { new MemReqCmd } + val replay_s2 = RegReset(Bool(false)) + val s2_valid = RegReset(Bool(false)) + val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) - val s1 = Reg() { new MemReqCmd } + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } s2_valid := s1_valid replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) + val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix + val repl_tag = s2_tags(repl_way).toUInt val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) tags.io.in.bits.rw := tag_we @@ -440,16 +442,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_data <> writeback.io.mem.req_data } -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) require(isPow2(entries) && entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = Reg(resetVal = Bool(false)) + val maybe_full = RegReset(Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -457,37 +459,37 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component val ptr_match = enq_ptr === deq_ptr val empty = ptr_match && !maybe_full val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) do_flow := empty && io.deq.ready - val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} - val ram_addr = Reg{Bits()} - val ram_out_valid = Reg{Bool()} + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) } -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) - val fq = new HellaFlowQueue(entries)(data) + val fq = Module(new HellaFlowQueue(entries)(data)) io.enq <> fq.io.enq io.deq <> Queue(fq.io.deq, 1, pipe = true) } object HellaQueue { - def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { - val q = (new HellaQueue(entries)) { enq.bits.clone } + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) q.io.enq.valid := enq.valid // not using <> so that override is allowed q.io.enq.bits := enq.bits enq.ready := q.io.enq.ready @@ -495,7 +497,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -507,17 +509,17 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component val inc = Bool() val dec = Bool() - val count = Reg(resetVal = UFix(numEntries, size)) - val watermark = count >= UFix(refillCycles) + val count = RegReset(UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) when (inc && !dec) { - count := count + UFix(1) + count := count + UInt(1) } when (!inc && dec) { - count := count - UFix(refillCycles) + count := count - UInt(refillCycles) } when (inc && dec) { - count := count - UFix(refillCycles-1) + count := count - UInt(refillCycles-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -528,7 +530,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component io.mem.req_data <> io.cpu.req_data - val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) resp_dataq.io.enq <> io.mem.resp io.cpu.resp <> resp_dataq.io.deq diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..9badb5e9 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,40 +2,45 @@ package uncore import Chisel._ import scala.math._ -class MemData extends Bundle { +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends Bundle { - val addr = UFix(width = MEM_ADDR_BITS) +trait HasMemAddr extends Bundle { + val addr = UInt(width = MEM_ADDR_BITS) +} + +trait HasMemTag extends Bundle { + val tag = UInt(width = MEM_TAG_BITS) +} + +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends Bundle { - val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) -} +class MemResp extends HasMemData with HasMemTag + +class MemData extends HasMemData class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp).flip } class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Valid(new MemResp).flip } class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = w) } - val resp = (new PipeIO) { Bits(width = w) }.flip + val req = Decoupled(Bits(width = w)) + val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Component +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new ioMem().flip @@ -45,19 +50,19 @@ class MemSerdes(w: Int) extends Component val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val out_buf = Reg() { Bits() } - val in_buf = Reg() { Bits() } + val out_buf = Reg(Bits()) + val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val state = RegReset(s_idle) + val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + send_cnt := send_cnt + UInt(1) + out_buf := out_buf >> UInt(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -76,31 +81,31 @@ class MemSerdes(w: Int) extends Component } when (state === s_read_addr && adone) { state := s_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_addr && adone) { state := s_write_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_idle && io.wide.req_data.valid) { state := s_write_data } when (state === s_write_data && ddone) { - data_send_cnt := data_send_cnt + UFix(1) - state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) - send_cnt := UFix(0) + data_send_cnt := data_send_cnt + UInt(1) + state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UInt(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val resp_val = Reg(resetVal = Bool(false)) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = RegReset(Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) + when (recv_cnt === UInt((rbits-1)/w)) { + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) resp_val := Bool(true) } in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) @@ -108,7 +113,7 @@ class MemSerdes(w: Int) extends Component io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits.data := in_buf >> UInt(io.wide.resp.bits.tag.width) } class MemDesserIO(w: Int) extends Bundle { @@ -116,7 +121,7 @@ class MemDesserIO(w: Int) extends Bundle { val wide = new ioMem } -class MemDesser(w: Int) extends Component // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -124,59 +129,59 @@ class MemDesser(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_cmd_recv) + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val state = RegReset(s_cmd_recv) - val in_buf = Reg() { Bits() } + val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv when (state === s_cmd_recv && adone) { state := s_cmd - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_cmd && io.wide.req_cmd.ready) { state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) } when (state === s_data_recv && ddone) { state := s_data - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - data_recv_cnt := data_recv_cnt + UFix(1) + data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UInt(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } + val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index cdaa8118..29c1dee8 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -3,24 +3,24 @@ import Chisel._ import scala.collection.mutable.Stack class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) + val meta = Decoupled(m) + val data = Decoupled(d) override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) + val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -28,7 +28,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data val grant_chosen_meta = !(locked && chosen_meta_has_data) (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits @@ -37,11 +37,11 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) + val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next - when(cnt_next === UFix(0)) { + when(cnt_next === UInt(0)) { locked := Bool(false) } } @@ -49,20 +49,20 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { when(!locked) { locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} } } } } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + val last_grant = RegReset(Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) var choose = Bits(n-1) for (i <- n-2 to 0 by -1) choose = Mux(io.in(i).meta.valid, Bits(i), choose) for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) meta_chosen := choose when (io.out.meta.fire()) { last_grant := meta_chosen } @@ -70,23 +70,23 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} } - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -99,8 +99,8 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -109,22 +109,22 @@ class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkC override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} } - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -135,30 +135,30 @@ class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: Physic case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + val in = Decoupled(data).flip + val out = Decoupled((new LogicalNetworkIO){data}) } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits @@ -168,18 +168,18 @@ class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T) } object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } + val in = Decoupled((new LogicalNetworkIO){data}).flip + val out = Decoupled(data) } io.out.valid := io.in.valid io.out.bits := io.in.bits.payload diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index b56b1396..4d42fccf 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,71 +1,68 @@ package uncore import Chisel._ -class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module { val io = new Bundle { - val out_fast = new FIFOIO()(data).flip - val out_slow = new FIFOIO()(data) - - val in_fast = new FIFOIO()(data) - val in_slow = new FIFOIO()(data).flip - + val out_fast = Decoupled(data).flip + val out_slow = Decoupled(data) + val in_fast = Decoupled(data) + val in_slow = Decoupled(data).flip val clk_slow = Bool(OUTPUT) - - val set_divisor = new PipeIO()(Bits(width = 32)).flip + val set_divisor = Valid(Bits(width = 32)).flip val divisor = Bits(OUTPUT, 32) } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = Reg(resetVal = UFix(divisor_max-1)) - val d_shadow = Reg(resetVal = UFix(divisor_max-1)) - val hold = Reg(resetVal = UFix(divisor_max/4-1)) - val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + val divisor = RegReset(UInt(divisor_max-1)) + val d_shadow = RegReset(UInt(divisor_max-1)) + val hold = RegReset(UInt(divisor_max/4-1)) + val h_shadow = RegReset(UInt(divisor_max/4-1)) when (io.set_divisor.valid) { - d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix - h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt } - io.divisor := hold << UFix(16) | divisor + io.divisor := hold << UInt(16) | divisor - val count = Reg{UFix(width = log2Up(divisor_max))} - val clock = Reg{Bool()} - count := count + UFix(1) + val count = Reg{UInt(width = log2Up(divisor_max))} + val myclock = Reg{Bool()} + count := count + UInt(1) - val rising = count === (divisor >> UFix(1)) + val rising = count === (divisor >> UInt(1)) val falling = count === divisor - val held = count === (divisor >> UFix(1)) + hold + val held = count === (divisor >> UInt(1)) + hold when (falling) { divisor := d_shadow hold := h_shadow - count := UFix(0) - clock := Bool(false) + count := UInt(0) + myclock := Bool(false) } when (rising) { - clock := Bool(true) + myclock := Bool(true) } - val in_slow_rdy = Reg(resetVal = Bool(false)) - val out_slow_val = Reg(resetVal = Bool(false)) - val out_slow_bits = Reg() { data } + val in_slow_rdy = RegReset(Bool(false)) + val out_slow_val = RegReset(Bool(false)) + val out_slow_bits = Reg(data) - val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) + val fromhost_q = Module(new Queue(data,1)) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new Queue(1)(data) + val tohost_q = Module(new Queue(data,1)) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := clock + io.clk_slow := myclock } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 6f7e6175..c16801ed 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,10 +5,11 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration + // TODO: override clone here, passing conf } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -29,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -39,7 +40,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -49,7 +50,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -61,7 +62,7 @@ object Acquire } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeBits) + val a_type = Bits(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -74,13 +75,13 @@ class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeBits) + val p_type = Bits(width = conf.co.probeTypeWidth) override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -90,7 +91,7 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeBits) + val r_type = Bits(width = conf.co.releaseTypeWidth) override def clone = { (new Release).asInstanceOf[this.type] } } @@ -99,7 +100,7 @@ class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeBits) + val g_type = Bits(width = conf.co.grantTypeWidth) override def clone = { (new Grant).asInstanceOf[this.type] } } @@ -111,13 +112,13 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } } class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { flip() override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } } @@ -142,25 +143,25 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI /* * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UFix + def arbIdx(in: Grant): UInt } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -170,14 +171,14 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -186,18 +187,18 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UFix = in.client_xact_id + def arbIdx(in: Grant): UInt = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -207,14 +208,14 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -223,18 +224,18 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) - def grantClientXactId(in: Grant) = UFix(0) // DNC + def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) + def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -244,14 +245,14 @@ class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConf req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 60e100e0..dde4190a 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,11 +1,11 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput + val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput } } @@ -17,8 +17,8 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Create SHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -28,7 +28,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client alloc_arb.io.in(i).valid := t.acquire.meta.ready @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -52,21 +52,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases t.release.data.bits := release.data.bits t.release.data.valid := release.data.valid } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } @@ -77,13 +77,13 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -95,13 +95,13 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = RegReset(Bool(false)) + val mem_cmd_sent = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -110,23 +110,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.master.acquire.meta.valid := Bool(false) io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) + io.master.acquire.data.bits.payload.data := UInt(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UFix(bankId) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) - io.client.grant.bits.payload.data := UFix(0) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) + io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.bits.header.src := UInt(bankId) io.client.grant_ack.valid := Bool(false) switch (state) { @@ -136,7 +136,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe xact := io.client.release.meta.bits.payload init_client_id_ := io.client.release.meta.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UFix(0) + mem_cnt := UInt(0) mem_cmd_sent := Bool(false) state := s_mem } @@ -159,56 +159,54 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) + val probe_flags = RegReset(Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val x_needs_read = RegReset(Bool(false)) + val acquire_data_needs_write = RegReset(Bool(false)) + val release_data_needs_write = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val a_w_mem_cmd_sent = RegReset(Bool(false)) + val r_w_mem_cmd_sent = RegReset(Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header := UFix(bankId) + io.master.acquire.data.bits.payload.data := UInt(0) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) + io.client.probe.bits.payload.master_xact_id := UInt(trackerId) io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UFix(0) - io.client.probe.bits.header.src := UFix(bankId) + io.client.probe.bits.header.dst := UInt(0) + io.client.probe.bits.header.src := UInt(bankId) io.client.grant.bits.payload.data := io.master.grant.bits.payload.data io.client.grant.bits.payload.g_type := grant_type io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) - io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.release.meta.ready := Bool(false) @@ -222,11 +220,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) probe_flags := probe_initial_flags - mem_cnt := UFix(0) + mem_cnt := UInt(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { @@ -241,12 +239,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.probe.bits.header.dst := curr_p_id } when(io.client.probe.ready) { - probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UFix(1) - when(release_count === UFix(1)) { + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { state := s_mem } when( co.messageHasData(io.client.release.meta.bits.payload)) { @@ -281,7 +279,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } @@ -290,7 +288,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } @@ -298,10 +296,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UInt(1) - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload @@ -315,7 +313,7 @@ abstract trait OuterRequestGenerator { when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { + when(mem_cnt === UInt(REFILL_CYCLES-1)) { trigger := Bool(false) } } From d5c9eb0f54e7e27f0d07126c79dbc6c1e701a6c5 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 12 Aug 2013 20:52:18 -0700 Subject: [PATCH 187/374] reset -> resetVal, getReset -> reset --- uncore/src/llc.scala | 2 +- uncore/src/slowio.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..db49f027 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..b8be1ed7 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 9162fbc9b5dde864c85a7e0555eeccd05c84f866 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 23:15:54 -0700 Subject: [PATCH 188/374] Clean up cloning in tilelink bundles --- uncore/src/tilelink.scala | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16801ed..fdd65094 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,7 +5,7 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration - // TODO: override clone here, passing conf + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { @@ -30,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -40,24 +40,15 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0) acq } } @@ -66,17 +57,12 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) - override def clone = { (new Acquire).asInstanceOf[this.type] } } - -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new AcquireData).asInstanceOf[this.type] } -} +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = conf.co.probeTypeWidth) - override def clone = { (new Probe).asInstanceOf[this.type] } } object Release @@ -92,22 +78,15 @@ object Release } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = conf.co.releaseTypeWidth) - override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new ReleaseData).asInstanceOf[this.type] } -} +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = conf.co.grantTypeWidth) - override def clone = { (new Grant).asInstanceOf[this.type] } -} - -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { - override def clone = { (new GrantAck).asInstanceOf[this.type] } } +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO From 7ff4126d049449aec3e4ff4e20fab8d2f08cd049 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 00:01:11 -0700 Subject: [PATCH 189/374] Abstracted UncachedTileLinkIOArbiters --- uncore/src/tilelink.scala | 136 ++++++++++---------------------------- 1 file changed, 35 insertions(+), 101 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index fdd65094..6a3c8e20 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -120,123 +120,57 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -/* - * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits def arbIdx(in: Grant): UInt -} -*/ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + val io = new Bundle { + val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UInt = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } - From 1308c08baa0eccdd98c3ec022a9de6217cc11cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 17:52:53 -0700 Subject: [PATCH 190/374] Reg standardization --- uncore/src/llc.scala | 2 +- uncore/src/network.scala | 42 ++++++++++++---------------------------- uncore/src/slowio.scala | 4 ++-- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..cd4341bb 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 29c1dee8..8733e2a6 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -148,43 +148,25 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled(data).flip - val out = Decoupled((new LogicalNetworkIO){data}) - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} object FIFOedLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled(in.bits.payload.clone).asDirectionless + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled((new LogicalNetworkIO){data}).flip - val out = Decoupled(data) - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..93c1324a 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 17d404b325fa71f715f03356baa060386840a41d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:27:38 -0700 Subject: [PATCH 191/374] final Reg changes --- uncore/src/llc.scala | 28 ++++++++++++++-------------- uncore/src/memserdes.scala | 18 +++++++++--------- uncore/src/network.scala | 8 ++++---- uncore/src/slowio.scala | 12 ++++++------ uncore/src/uncore.scala | 32 ++++++++++++++++---------------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cd4341bb..f9907934 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -29,7 +29,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = RegUpdate(ren) + val reg_ren = Reg(next=ren) val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) @@ -113,7 +113,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec.fill(outstanding){RegReset(Bool(false))} + val valid = Vec.fill(outstanding){Reg(init=Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) val mshr = Vec.fill(outstanding){Reg(new MSHR)} @@ -191,12 +191,12 @@ class LLCWriteback(requestors: Int) extends Module val mem = new ioMemPipe } - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val who = Reg(UInt()) val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -252,10 +252,10 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul } val q = Module(new Queue(new QEntry, latency+2)) val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) @@ -307,7 +307,7 @@ class MemReqArb(n: Int) extends Module // UNTESTED val mem = new ioMem } - val lock = RegReset(Bool(false)) + val lock = Reg(init=Bool(false)) val locker = Reg(UInt()) val arb = Module(new RRArbiter(new MemReqCmd, n)) @@ -360,17 +360,17 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val data = Module(new LLCData(4, sets, ways, dataLeaf)) val writeback = Module(new LLCWriteback(2)) - val initCount = RegReset(UInt(0, log2Up(sets+1))) + val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = RegReset(Bool(false)) - val s2_valid = RegReset(Bool(false)) + val replay_s2 = Reg(init=Bool(false)) + val s2_valid = Reg(init=Bool(false)) val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) + val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } @@ -451,7 +451,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = RegReset(Bool(false)) + val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -509,7 +509,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module val inc = Bool() val dec = Bool() - val count = RegReset(UInt(numEntries, size)) + val count = Reg(init=UInt(numEntries, size)) val watermark = count >= UInt(refillCycles) when (inc && !dec) { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 9badb5e9..7a021621 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -54,9 +54,9 @@ class MemSerdes(w: Int) extends Module val in_buf = Reg(Bits()) val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } - val state = RegReset(s_idle) - val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val state = Reg(init=s_idle) + val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -96,9 +96,9 @@ class MemSerdes(w: Int) extends Module send_cnt := UInt(0) } - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val resp_val = RegReset(Bool(false)) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { @@ -129,14 +129,14 @@ class MemDesser(w: Int) extends Module // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } - val state = RegReset(s_cmd_recv) + val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 8733e2a6..9de890b1 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -19,8 +19,8 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) - val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) + val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -37,7 +37,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt = Reg(init=UInt(0, width = log2Up(count))) val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next @@ -54,7 +54,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock } } } - val last_grant = RegReset(Bits(0, log2Up(n))) + val last_grant = Reg(init=Bits(0, log2Up(n))) val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 93c1324a..79c837d1 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -14,10 +14,10 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = RegReset(UInt(divisor_max-1)) - val d_shadow = RegReset(UInt(divisor_max-1)) - val hold = RegReset(UInt(divisor_max/4-1)) - val h_shadow = RegReset(UInt(divisor_max/4-1)) + val divisor = Reg(init=UInt(divisor_max-1)) + val d_shadow = Reg(init=UInt(divisor_max-1)) + val hold = Reg(init=UInt(divisor_max/4-1)) + val h_shadow = Reg(init=UInt(divisor_max/4-1)) when (io.set_divisor.valid) { d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt @@ -42,8 +42,8 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module myclock := Bool(true) } - val in_slow_rdy = RegReset(Bool(false)) - val out_slow_val = RegReset(Bool(false)) + val in_slow_rdy = Reg(init=Bool(false)) + val out_slow_val = Reg(init=Bool(false)) val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dde4190a..bfc703fa 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -96,11 +96,11 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = RegReset(Bool(false)) - val mem_cmd_sent = RegReset(Bool(false)) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = Reg(init=Bool(false)) + val mem_cmd_sent = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) @@ -160,23 +160,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) - val probe_flags = RegReset(Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = RegReset(Bool(false)) - val acquire_data_needs_write = RegReset(Bool(false)) - val release_data_needs_write = RegReset(Bool(false)) + val x_needs_read = Reg(init=Bool(false)) + val acquire_data_needs_write = Reg(init=Bool(false)) + val release_data_needs_write = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = RegReset(Bool(false)) - val r_w_mem_cmd_sent = RegReset(Bool(false)) + val a_w_mem_cmd_sent = Reg(init=Bool(false)) + val r_w_mem_cmd_sent = Reg(init=Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { @@ -296,7 +296,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UInt(1) def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { From 3763cd00049bfad25d1857223af32df504042f77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:57:16 -0700 Subject: [PATCH 192/374] standardizing sbt build conventions --- uncore/README | 6 ++++++ uncore/build.sbt | 14 ++++++++++++++ uncore/src/{ => main/scala}/coherence.scala | 0 uncore/src/{ => main/scala}/consts.scala | 0 uncore/src/{ => main/scala}/llc.scala | 0 uncore/src/{ => main/scala}/memserdes.scala | 0 uncore/src/{ => main/scala}/network.scala | 0 uncore/src/{ => main/scala}/package.scala | 0 uncore/src/{ => main/scala}/slowio.scala | 0 uncore/src/{ => main/scala}/tilelink.scala | 0 uncore/src/{ => main/scala}/uncore.scala | 0 11 files changed, 20 insertions(+) create mode 100644 uncore/README create mode 100644 uncore/build.sbt rename uncore/src/{ => main/scala}/coherence.scala (100%) rename uncore/src/{ => main/scala}/consts.scala (100%) rename uncore/src/{ => main/scala}/llc.scala (100%) rename uncore/src/{ => main/scala}/memserdes.scala (100%) rename uncore/src/{ => main/scala}/network.scala (100%) rename uncore/src/{ => main/scala}/package.scala (100%) rename uncore/src/{ => main/scala}/slowio.scala (100%) rename uncore/src/{ => main/scala}/tilelink.scala (100%) rename uncore/src/{ => main/scala}/uncore.scala (100%) diff --git a/uncore/README b/uncore/README new file mode 100644 index 00000000..c920e19a --- /dev/null +++ b/uncore/README @@ -0,0 +1,6 @@ +This is the repository for uncore components assosciated with UCB_BAR chip +projects. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. +lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + diff --git a/uncore/build.sbt b/uncore/build.sbt new file mode 100644 index 00000000..d5f2e6ea --- /dev/null +++ b/uncore/build.sbt @@ -0,0 +1,14 @@ +organization := "edu.berkeley.cs" + +version := "2.0" + +name := "uncore" + +scalaVersion := "2.10.2" + +resolvers ++= Seq( + "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" +) + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" diff --git a/uncore/src/coherence.scala b/uncore/src/main/scala/coherence.scala similarity index 100% rename from uncore/src/coherence.scala rename to uncore/src/main/scala/coherence.scala diff --git a/uncore/src/consts.scala b/uncore/src/main/scala/consts.scala similarity index 100% rename from uncore/src/consts.scala rename to uncore/src/main/scala/consts.scala diff --git a/uncore/src/llc.scala b/uncore/src/main/scala/llc.scala similarity index 100% rename from uncore/src/llc.scala rename to uncore/src/main/scala/llc.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/main/scala/memserdes.scala similarity index 100% rename from uncore/src/memserdes.scala rename to uncore/src/main/scala/memserdes.scala diff --git a/uncore/src/network.scala b/uncore/src/main/scala/network.scala similarity index 100% rename from uncore/src/network.scala rename to uncore/src/main/scala/network.scala diff --git a/uncore/src/package.scala b/uncore/src/main/scala/package.scala similarity index 100% rename from uncore/src/package.scala rename to uncore/src/main/scala/package.scala diff --git a/uncore/src/slowio.scala b/uncore/src/main/scala/slowio.scala similarity index 100% rename from uncore/src/slowio.scala rename to uncore/src/main/scala/slowio.scala diff --git a/uncore/src/tilelink.scala b/uncore/src/main/scala/tilelink.scala similarity index 100% rename from uncore/src/tilelink.scala rename to uncore/src/main/scala/tilelink.scala diff --git a/uncore/src/uncore.scala b/uncore/src/main/scala/uncore.scala similarity index 100% rename from uncore/src/uncore.scala rename to uncore/src/main/scala/uncore.scala From dc53529156d54eba953640e71f567c5bb9644cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:00:51 -0700 Subject: [PATCH 193/374] added resolver, bumped chisel dependency --- uncore/build.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index d5f2e6ea..abe95365 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -8,7 +8,9 @@ scalaVersion := "2.10.2" resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", + "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" + ) -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From 9aff60f340bb3d64ed7a1bf482cf80b93714cb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:16:42 -0700 Subject: [PATCH 194/374] whitespace error in build.sbt --- uncore/build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index abe95365..6ea793a4 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -10,7 +10,6 @@ resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" - ) libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From b01fe4f6aa80e7b452910e22bdbbb996a4d26cc5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 24 Aug 2013 15:24:17 -0700 Subject: [PATCH 195/374] fix memserdes bit ordering --- uncore/src/memserdes.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..2af04df8 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -107,8 +107,7 @@ class MemSerdes(w: Int) extends Component } io.wide.resp.valid := resp_val - io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } class MemDesserIO(w: Int) extends Bundle { From ee98cd8378c8b97e141f25390bb2f7cd1803e81d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 10:54:51 -0700 Subject: [PATCH 196/374] new enum syntax --- uncore/src/main/scala/coherence.scala | 66 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 4 +- uncore/src/main/scala/uncore.scala | 4 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7109be94..dcb80c8e 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -96,10 +96,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -156,13 +156,13 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -298,13 +298,13 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -456,13 +456,13 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -617,13 +617,13 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -782,12 +782,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 39c30e56..42611bc6 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -53,7 +53,7 @@ class MemSerdes(w: Int) extends Module val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) @@ -134,7 +134,7 @@ class MemDesser(w: Int) extends Module // test rig side val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(UInt(), 5) val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index bfc703fa..adf5f638 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -95,7 +95,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) @@ -159,7 +159,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) From 1cac26fd76f7eea8668420d7d63903b42653ccb8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 16:15:41 -0700 Subject: [PATCH 197/374] NetworkIOs no longer use thunks --- uncore/src/main/scala/network.scala | 54 ++++++++++++++-------------- uncore/src/main/scala/tilelink.scala | 30 ++++++++-------- uncore/src/main/scala/uncore.scala | 4 +-- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 9de890b1..4905ffde 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -2,23 +2,23 @@ package uncore import Chisel._ import scala.collection.mutable.Stack -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = Decoupled(m) - val data = Decoupled(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { + val meta = Decoupled(mType) + val data = Decoupled(dType) + override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } } -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { + val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip + val out = new PairedDataIO(mType, dType) + val meta_chosen = UInt(OUTPUT, log2Up(n)) + val data_chosen = UInt(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { +class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) + val io = new PairedArbiterIO(mType, dType, n) val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) @@ -68,10 +68,10 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} } val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -79,7 +79,7 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) + val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits @@ -103,18 +103,18 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UInt(width = conf.idBits) } -class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data - override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = new PhysicalHeader + val payload = dType.clone + override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } } abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} + val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} } val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -148,7 +148,7 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits out.bits.header.dst := dst @@ -168,8 +168,8 @@ object FIFOedLogicalNetworkIOUnwrapper { } } -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6a3c8e20..425cbccd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -91,32 +91,32 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { +class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } } class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) - val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) - val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) + val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) + val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) - val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) + val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) + val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -131,7 +131,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -141,7 +141,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur req.meta.ready := arb.meta.ready }} - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index adf5f638..a24f309d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -66,7 +66,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } From cc7783404df6dd87b7eaef3aa93b2b80906e8a13 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 12 Sep 2013 16:09:53 -0700 Subject: [PATCH 198/374] Add memory command M_XA_XOR --- uncore/src/main/scala/consts.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 15bfee4d..3a1a8482 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -21,13 +21,14 @@ trait MemoryOpConstants { val M_XWR = Bits("b00001"); // int store val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_XA_SWAP = Bits("b00100"); val M_FENCE = Bits("b00101"); // memory fence val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); - val M_XA_SWAP = Bits("b01001"); - val M_XA_AND = Bits("b01010"); - val M_XA_OR = Bits("b01011"); + val M_XA_XOR = Bits("b01001"); + val M_XA_OR = Bits("b01010"); + val M_XA_AND = Bits("b01011"); val M_XA_MIN = Bits("b01100"); val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); @@ -35,7 +36,7 @@ trait MemoryOpConstants { val M_INV = Bits("b10000"); // write back and invalidate line val M_CLN = Bits("b10001"); // write back line - def isAMO(cmd: Bits) = cmd(3) + def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) From 9bf10ae5d22ebcc927703d43842097867add7ea1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 19 Sep 2013 15:26:36 -0700 Subject: [PATCH 199/374] remove extraneous toBits (need new Chisel) --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From cc3dc1bd0fbed6aad7f8ba9ade655f80f5e12369 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 19 Sep 2013 20:10:56 -0700 Subject: [PATCH 200/374] bug fix --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From 20246b373ebfdb265c7bc095364384f1caee028b Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 24 Sep 2013 16:02:00 -0700 Subject: [PATCH 201/374] Properly ignore target files --- uncore/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 uncore/.gitignore diff --git a/uncore/.gitignore b/uncore/.gitignore new file mode 100644 index 00000000..555feb41 --- /dev/null +++ b/uncore/.gitignore @@ -0,0 +1,2 @@ +target/ +project/target/ From 42693d43adc3dbbb65b1b0a591d3a20db904a804 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Sep 2013 09:51:14 -0700 Subject: [PATCH 202/374] simplify build.sbt --- uncore/build.sbt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 6ea793a4..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,11 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -resolvers ++= Seq( - "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", - "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" -) - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From f440df5338d3c1e7ba56fcd57ca8fa2ae98f5f99 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 28 Oct 2013 22:37:41 -0700 Subject: [PATCH 203/374] rename M_FENCE to M_NOP --- uncore/src/main/scala/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 3a1a8482..e9d282ae 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -22,7 +22,7 @@ trait MemoryOpConstants { val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write val M_XA_SWAP = Bits("b00100"); - val M_FENCE = Bits("b00101"); // memory fence + val M_NOP = Bits("b00101"); val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); From c350cbd6eae0153ed511ceac6b45a9476d52a965 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 13:19:04 -0800 Subject: [PATCH 204/374] move htif to uncore --- uncore/src/main/scala/htif.scala | 267 +++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 uncore/src/main/scala/htif.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala new file mode 100644 index 00000000..38fce75e --- /dev/null +++ b/uncore/src/main/scala/htif.scala @@ -0,0 +1,267 @@ +package rocket + +import Chisel._ +import Node._ +import uncore._ + +class HostIO(val w: Int) extends Bundle +{ + val clk = Bool(OUTPUT) + val clk_edge = Bool(OUTPUT) + val in = Decoupled(Bits(width = w)).flip + val out = Decoupled(Bits(width = w)) + val debug_stats_pcr = Bool(OUTPUT) +} + +class PCRReq extends Bundle +{ + val rw = Bool() + val addr = Bits(width = 5) + val data = Bits(width = 64) +} + +class HTIFIO(ntiles: Int) extends Bundle +{ + val reset = Bool(INPUT) + val id = UInt(INPUT, log2Up(ntiles)) + val pcr_req = Decoupled(new PCRReq).flip + val pcr_rep = Decoupled(Bits(width = 64)) + val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_rep = Decoupled(Bool()).flip + val debug_stats_pcr = Bool(OUTPUT) + // wired directly to stats register + // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work +} + +class SCRIO(n: Int) extends Bundle +{ + val rdata = Vec.fill(n){Bits(INPUT, 64)} + val wen = Bool(OUTPUT) + val waddr = UInt(OUTPUT, log2Up(n)) + val wdata = Bits(OUTPUT, 64) +} + +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +{ + implicit val (ln, co) = (conf.ln, conf.co) + val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val io = new Bundle { + val host = new HostIO(w) + val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val mem = new TileLinkIO + val scr = new SCRIO(nSCR) + } + + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) + // system is 'interesting' if any tile is 'interesting' + + val short_request_bits = 64 + val long_request_bits = 576 + require(short_request_bits % w == 0) + + val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits + val rx_count = Reg(init=UInt(0,rx_count_w)) + val rx_shifter = Reg(Bits(width = short_request_bits)) + val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) + val next_cmd = rx_shifter_in(3,0) + val cmd = Reg(Bits()) + val size = Reg(Bits()) + val pos = Reg(Bits()) + val seqno = Reg(Bits()) + val addr = Reg(Bits()) + when (io.host.in.valid && io.host.in.ready) { + rx_shifter := rx_shifter_in + rx_count := rx_count + UInt(1) + when (rx_count === UInt(short_request_bits/w-1)) { + cmd := next_cmd + size := rx_shifter_in(15,4) + pos := rx_shifter_in(15,4+OFFSET_BITS-3) + seqno := rx_shifter_in(23,16) + addr := rx_shifter_in(63,24) + } + } + + val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) + val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR + val packet_ram_depth = long_request_bits/short_request_bits-1 + val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + when (rx_word_done && io.host.in.ready) { + packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in + } + + val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) + + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_wdata = packet_ram(0) + + val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), + Bool(true))) + + val tx_count = Reg(init=UInt(0, rx_count_w)) + val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) + val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) + val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) + when (io.host.out.valid && io.host.out.ready) { + tx_count := tx_count + UInt(1) + } + + val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd != cmd_writemem && next_cmd != cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) + val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) + val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) + + val mem_acked = Reg(init=Bool(false)) + val mem_gxid = Reg(Bits()) + val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_needs_ack = Reg(Bool()) + when (io.mem.grant.valid) { + mem_acked := Bool(true) + mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gsrc := io.mem.grant.bits.header.src + mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + } + io.mem.grant.ready := Bool(true) + + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state = Reg(init=state_rx) + + val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) + when (state === state_rx && rx_done) { + state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, + state_tx)) + } + + val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val x_init = Module(new Queue(new Acquire, 1)) + when (state === state_mem_req && x_init.io.enq.ready) { + state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + } + when (state === state_mem_wdata && io.mem.acquire.data.ready) { + when (mem_cnt.andR) { + state := state_mem_wresp + } + mem_cnt := mem_cnt + UInt(1) + } + when (state === state_mem_wresp) { + when (mem_acked) { + state := state_mem_finish + mem_acked := Bool(false) + } + } + when (state === state_mem_rdata) { + when (io.mem.grant.valid) { + when (mem_cnt.andR) { + state := state_mem_finish + } + mem_cnt := mem_cnt + UInt(1) + } + mem_acked := Bool(false) + } + when (state === state_mem_finish && io.mem.grant_ack.ready) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << OFFSET_BITS-3) + } + when (state === state_tx && tx_done) { + when (tx_word_count === tx_size) { + rx_count := UInt(0) + tx_count := UInt(0) + } + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + } + + var mem_req_data: Bits = null + for (i <- 0 until MEM_DATA_BITS/short_request_bits) { + val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) + when (state === state_mem_rdata && io.mem.grant.valid) { + packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + } + mem_req_data = Cat(packet_ram(idx), mem_req_data) + } + x_init.io.enq.valid := state === state_mem_req + val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + x_init.io.enq.bits := Mux(cmd === cmd_writemem, + Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), + Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id + io.mem.acquire.data.valid := state === state_mem_wdata + io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid + io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.probe.ready := Bool(false) + io.mem.release.meta.valid := Bool(false) + io.mem.release.data.valid := Bool(false) + + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) + for (i <- 0 until nTiles) { + val my_reset = Reg(init=Bool(true)) + val my_ipi = Reg(init=Bool(false)) + + val cpu = io.cpu(i) + val me = pcr_coreid === UInt(i) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.bits.rw := cmd === cmd_writecr + cpu.pcr_req.bits.addr := pcr_addr + cpu.pcr_req.bits.data := pcr_wdata + cpu.reset := my_reset + + when (cpu.ipi_rep.ready) { + my_ipi := Bool(false) + } + cpu.ipi_rep.valid := my_ipi + cpu.ipi_req.ready := Bool(true) + for (j <- 0 until nTiles) { + when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { + my_ipi := Bool(true) + } + } + + when (cpu.pcr_req.valid && cpu.pcr_req.ready) { + state := state_pcr_resp + } + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (cmd === cmd_writecr) { + my_reset := pcr_wdata(0) + } + pcrReadData := my_reset.toBits + state := state_tx + } + + cpu.pcr_rep.ready := Bool(true) + when (cpu.pcr_rep.valid) { + pcrReadData := cpu.pcr_rep.bits + state := state_tx + } + } + + val scr_addr = addr(log2Up(nSCR)-1, 0) + val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} + for (i <- 0 until scr_rdata.size) + scr_rdata(i) := io.scr.rdata(i) + scr_rdata(0) := UInt(nTiles) + scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + + io.scr.wen := Bool(false) + io.scr.wdata := pcr_wdata + io.scr.waddr := scr_addr.toUInt + when (state === state_pcr_req && pcr_coreid === SInt(-1)) { + io.scr.wen := cmd === cmd_writecr + pcrReadData := scr_rdata(scr_addr) + state := state_tx + } + + val tx_cmd = Mux(nack, cmd_nack, cmd_ack) + val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) + val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) + val tx_data = Mux(tx_word_count === UInt(0), tx_header, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, pcrReadData, + packet_ram(packet_ram_raddr))) + + io.host.in.ready := state === state_rx + io.host.out.valid := state === state_tx + io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) +} From f13d76628b122d2dfc3950aa0761bcfd0f7394fc Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 15:42:10 -0800 Subject: [PATCH 205/374] forgot to put htif into uncore package --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 38fce75e..ec905012 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ From 056bb156cab81319750c7de53a55af57dfee7599 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 20 Nov 2013 16:43:55 -0800 Subject: [PATCH 206/374] make CacheConstants an object --- uncore/src/main/scala/consts.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index e9d282ae..bbe6cbd5 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -54,6 +54,7 @@ trait AddressConstants { val PERM_BITS = 6; } +object CacheConstants extends CacheConstants trait CacheConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) From acc0d2b06c6007993552cf949fcd3af809b5ce54 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 25 Nov 2013 04:34:16 -0800 Subject: [PATCH 207/374] Only use LSBs for HTIF control regs For now, at least... --- uncore/src/main/scala/htif.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index ec905012..cd5c7534 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -196,6 +196,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati io.mem.release.meta.valid := Bool(false) io.mem.release.data.valid := Bool(false) + val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nTiles) { val my_reset = Reg(init=Bool(true)) @@ -203,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -223,7 +224,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (state === state_pcr_req && me && pcr_addr === pcr_reset) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From 4f1213cb8b638067702500c1d2c274595c690543 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 13 Jan 2014 21:45:14 -0800 Subject: [PATCH 208/374] Fix Scala integer overflow --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cd5c7534..fdd1162c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -244,7 +244,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata From 3e634aef1d3afedacc4035248d81bc742377aff5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 22 Jan 2014 18:20:36 -0800 Subject: [PATCH 209/374] Fix HTIF for cache line sizes other than 64 B --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index fdd1162c..4e894af8 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = 576 + val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits From bbf801023039b406f5c6ad2ce952441fcf29d644 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Jan 2014 12:20:55 -0800 Subject: [PATCH 210/374] cleanups supporting uncore hierarchy --- uncore/src/main/scala/coherence.scala | 244 ++++++++++++++------------ uncore/src/main/scala/consts.scala | 17 +- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 25 +-- uncore/src/main/scala/tilelink.scala | 192 +++++++++++++------- uncore/src/main/scala/uncore.scala | 8 +- 6 files changed, 284 insertions(+), 206 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index dcb80c8e..e0db85f1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,10 +1,6 @@ package uncore import Chisel._ -abstract trait CoherenceAgentRole -abstract trait ClientCoherenceAgent extends CoherenceAgentRole -abstract trait MasterCoherenceAgent extends CoherenceAgentRole - abstract class CoherencePolicy { def nClientStates: Int def nMasterStates: Int @@ -31,41 +27,42 @@ abstract class CoherencePolicy { def newStateOnWriteback(): UInt def newStateOnFlush(): UInt def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): Bits + def newStateOnProbe(incoming: Probe, state: UInt): UInt def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt def getProbeType(a_type: UInt, global_state: UInt): UInt - def getReleaseTypeOnCacheControl(cmd: UInt): Bits - def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits - def getGrantType(a_type: UInt, count: UInt): Bits - def getGrantType(rel: Release, count: UInt): Bits + def getReleaseTypeOnCacheControl(cmd: UInt): UInt + def getReleaseTypeOnVoluntaryWriteback(): UInt + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getGrantType(a_type: UInt, count: UInt): UInt + def getGrantType(rel: Release, count: UInt): UInt def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def needsAckReply(a_type: UInt, global_state: UInt): Bool - def needsSelfProbe(acq: Acquire): Bool - def requiresAck(grant: Grant): Bool - def requiresAck(release: Release): Bool + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool + def requiresSelfProbe(a_type: UInt): Bool + def requiresAckForGrant(g_type: UInt): Bool + def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) + def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { - def getUncachedReadAcquireType: Bits - def getUncachedWriteAcquireType: Bits - def getUncachedReadWordAcquireType: Bits - def getUncachedWriteWordAcquireType: Bits - def getUncachedAtomicAcquireType: Bits + def getUncachedReadAcquireType: UInt + def getUncachedWriteAcquireType: UInt + def getUncachedReadWordAcquireType: UInt + def getUncachedWriteWordAcquireType: UInt + def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool } @@ -73,18 +70,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) - def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): UInt = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) + def getGrantType(rel: Release, count: UInt): UInt = UInt(0) def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) - def requiresAck(grant: Grant) = Bool(true) - def requiresAck(release: Release) = Bool(false) + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g_type: UInt) = Bool(true) + def requiresAckForRelease(r_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -135,17 +133,17 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -202,7 +200,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -220,9 +218,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +233,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,7 +256,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -281,12 +279,15 @@ class MICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -350,7 +351,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -373,9 +374,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +391,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,7 +414,7 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -438,12 +439,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -514,7 +518,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -537,9 +541,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,19 +558,19 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -577,7 +581,7 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -599,12 +603,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -675,7 +682,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -698,9 +705,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,19 +722,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -738,7 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -763,13 +770,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -795,7 +805,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -807,8 +817,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } def needsWriteback (state: UInt): Bool = { @@ -846,7 +856,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -875,11 +885,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,19 +904,19 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, @@ -918,7 +928,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -944,12 +954,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index bbe6cbd5..d4f7130c 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -6,14 +6,15 @@ import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { - val MT_X = Bits("b???", 3); - val MT_B = Bits("b000", 3); - val MT_H = Bits("b001", 3); - val MT_W = Bits("b010", 3); - val MT_D = Bits("b011", 3); - val MT_BU = Bits("b100", 3); - val MT_HU = Bits("b101", 3); - val MT_WU = Bits("b110", 3); + val MT_SZ = 3 + val MT_X = Bits("b???") + val MT_B = Bits("b000") + val MT_H = Bits("b001") + val MT_W = Bits("b010") + val MT_D = Bits("b011") + val MT_BU = Bits("b100") + val MT_HU = Bits("b101") + val MT_WU = Bits("b110") val M_SZ = 5 val M_X = Bits("b?????"); diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4e894af8..608ac051 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -120,7 +120,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 4905ffde..613a3b65 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -68,11 +68,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, c when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} - } +} + +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new PairedCrossbarIO(mType, dType) val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -111,11 +113,12 @@ class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfi abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} - } +} +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new BasicCrossbarIO(dType) val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -133,14 +136,12 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) +case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { + val nEndpoints = nMasters + nClients } +abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module + class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val src = UInt(width = conf.idBits) val dst = UInt(width = conf.idBits) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 425cbccd..cd50213f 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -40,20 +40,25 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } + def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = new Acquire + acq := a + acq + } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeWidth) + val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -61,13 +66,29 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Probe +{ + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val prb = new Probe + prb.p_type := p_type + prb.addr := addr + prb.master_xact_id := master_xact_id + prb + } +} class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeWidth) + val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -77,100 +98,141 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = conf.co.releaseTypeWidth) } class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeWidth) + val g_type = UInt(width = conf.co.grantTypeWidth) } class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId -trait DirectionalIO -trait ClientSourcedIO extends DirectionalIO -trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} -class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) - val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) - val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) + val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) - val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - def acquireClientXactId(in: Acquire, id: Int): Bits - def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UInt - +abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + + type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId + type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData + + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def masterSourcedClientXactId(in: MasterSourcedWithId): Bits + def arbIdx(in: MasterSourcedWithId): UInt + + def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] + (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], + out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, + arbN, REFILL_CYCLES, hasData _)) + out <> arb.io.out + ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload := req.meta.bits.payload + arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + } + + def hookupMasterSource[M <: MasterSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { + out.ready := Bool(false) + for (i <- 0 until arbN) { + ins(i).valid := Bool(false) + when (arbIdx(out.bits.payload) === UInt(i)) { + ins(i).valid := out.valid + out.ready := ins(i).ready + } + ins(i).bits := out.bits + ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + } + } +} + +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupMasterSource(io.in.map(_.grant), io.out.grant) val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } +} - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) +abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { + val io = new Bundle { + val in = Vec.fill(n){new TileLinkIO}.flip + val out = new TileLinkIO } + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupMasterSource(io.in.map(_.grant), io.out.grant) + + io.in.map{ _.probe.valid := io.out.probe.valid } + io.in.map{ _.probe.bits := io.out.probe.bits } + io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) + + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt +abstract trait AppendsArbiterId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id >> UInt(log2Up(arbN)) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id(log2Up(arbN)-1,0).toUInt } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id - def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UInt = in.client_xact_id +abstract trait PassesId { + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + in.client_xact_id + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) - def grantClientXactId(in: Grant) = UInt(0) // DNC - def arbIdx(in: Grant) = in.client_xact_id +abstract trait UsesNewId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + UInt(id, log2Up(arbN)) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + UInt(0) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index a24f309d..8b410639 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO @@ -279,13 +279,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) + state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, + Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) } } is(s_ack) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } + when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { From 02dbd6b0aaaf0967ea26afbc2e4d32d12882d1ac Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 12 Feb 2014 18:39:40 -0800 Subject: [PATCH 211/374] Don't assign to your own inputs --- uncore/src/main/scala/uncore.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8b410639..67dfde85 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -127,7 +127,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { @@ -212,7 +211,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { From dec98eb047db3d5184961e7297b3728fe7be80a9 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Tue, 18 Mar 2014 11:37:08 -0700 Subject: [PATCH 212/374] Update library dependencies (jenkins builds) Add chisel as an explicit library dependency --- uncore/build.sbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/build.sbt b/uncore/build.sbt index 8794797d..565e5fc7 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,3 +5,5 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From a228dbfa0d76237258026ff08f76375edef6524b Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 20 Mar 2014 10:40:05 -0700 Subject: [PATCH 213/374] Revert "Update library dependencies (jenkins builds)" This reverts commit e7a12143d0bfe8b3b4a4dcc78d119a89553ade8c. --- uncore/build.sbt | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 565e5fc7..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,5 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From fbca7c6bb35914ac0c3f043090d4e8d407306ae2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 29 Mar 2014 10:53:49 -0700 Subject: [PATCH 214/374] refactor ioMem and associcated constants. merge Aqcuire and AcquireData --- uncore/src/main/scala/coherence.scala | 17 -- uncore/src/main/scala/consts.scala | 6 - uncore/src/main/scala/htif.scala | 56 +++-- uncore/src/main/scala/llc.scala | 72 +++--- uncore/src/main/scala/memserdes.scala | 141 +++++++++--- uncore/src/main/scala/network.scala | 113 +-------- uncore/src/main/scala/tilelink.scala | 89 ++++--- uncore/src/main/scala/uncore.scala | 320 +++++++++++--------------- 8 files changed, 384 insertions(+), 430 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index e0db85f1..c9d0f4eb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -48,7 +48,6 @@ abstract class CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -79,7 +78,6 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def requiresAckForGrant(g_type: UInt) = Bool(true) def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -282,9 +280,6 @@ class MICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -442,9 +437,6 @@ class MEICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -606,9 +598,6 @@ class MSICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -773,9 +762,6 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -958,9 +944,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index d4f7130c..b0cdcdd6 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -70,9 +70,3 @@ trait TileLinkSizeConstants { trait MemoryInterfaceConstants extends CacheConstants with AddressConstants -{ - val MEM_TAG_BITS = 5 - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS -} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 608ac051..c4d0ffb0 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES + val long_request_bits = short_request_bits + conf.dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -124,26 +124,23 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati } io.mem.grant.ready := Bool(true) - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) val state = Reg(init=state_rx) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { - state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, + Mux(rx_cmd === cmd_writemem, state_mem_wreq, Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, - state_tx)) + state_tx))) } - val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val x_init = Module(new Queue(new Acquire, 1)) - when (state === state_mem_req && x_init.io.enq.ready) { - state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + val acq_q = Module(new Queue(new Acquire, 1)) + when (state === state_mem_wreq && acq_q.io.enq.ready) { + state := state_mem_wresp } - when (state === state_mem_wdata && io.mem.acquire.data.ready) { - when (mem_cnt.andR) { - state := state_mem_wresp - } - mem_cnt := mem_cnt + UInt(1) + when (state === state_mem_rreq && acq_q.io.enq.ready) { + state := state_mem_rresp } when (state === state_mem_wresp) { when (mem_acked) { @@ -151,12 +148,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(false) } } - when (state === state_mem_rdata) { + when (state === state_mem_rresp) { when (io.mem.grant.valid) { - when (mem_cnt.andR) { - state := state_mem_finish - } - mem_cnt := mem_cnt + UInt(1) + state := state_mem_finish } mem_acked := Bool(false) } @@ -170,31 +164,33 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati rx_count := UInt(0) tx_count := UInt(0) } - state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } var mem_req_data: Bits = null - for (i <- 0 until MEM_DATA_BITS/short_request_bits) { - val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) - when (state === state_mem_rdata && io.mem.grant.valid) { + for (i <- 0 until conf.dataBits/short_request_bits) { + val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - x_init.io.enq.valid := state === state_mem_req + acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) - x_init.io.enq.bits := Mux(cmd === cmd_writemem, + acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) - io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id - io.mem.acquire.data.valid := state === state_mem_wdata - io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.acquire.valid := acq_q.io.deq.valid + acq_q.io.deq.ready := io.mem.acquire.ready + io.mem.acquire.bits.payload := acq_q.io.deq.bits + io.mem.acquire.bits.payload.data := mem_req_data + io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid io.mem.grant_ack.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) - io.mem.release.meta.valid := Bool(false) - io.mem.release.data.valid := Bool(false) + io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) @@ -244,7 +240,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 6627527a..1bfa4da7 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,42 +72,42 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int) extends MemReqCmd +class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] } -class LLCTagReq(ways: Int) extends HasMemAddr +class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) - val mem = new ioMemPipe + val mem = new MemPipeIO val mem_resp_set = UInt(OUTPUT, log2Up(sets)) val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UInt(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = conf.addrBits - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -139,7 +139,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -183,12 +183,12 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new ioMemPipe + val mem = new MemPipeIO } val valid = Reg(init=Bool(false)) @@ -196,7 +196,7 @@ class LLCWriteback(requestors: Int) extends Module val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -216,7 +216,7 @@ class LLCWriteback(requestors: Int) extends Module when (io.mem.req_data.valid && io.mem.req_data.ready) { count := count + UInt(1) - when (count === UInt(REFILL_CYCLES-1)) { + when (count === UInt(refill_cycles-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int) extends Module io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback = Decoupled(UInt(width = conf.addrBits)) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -254,12 +254,12 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val qReady = q.io.count <= UInt(q.entries-latency-1) val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) + val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) - when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } + when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } @@ -300,11 +300,11 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Module // UNTESTED +class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = Vec.fill(n){new ioMem().flip} - val mem = new ioMem + val cpu = Vec.fill(n){new MemIO().flip} + val mem = new MemIO } val lock = Reg(init=Bool(false)) @@ -339,26 +339,26 @@ class MemReqArb(n: Int) extends Module // UNTESTED } io.mem.resp.ready := io.cpu(respWho).resp.ready - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } - val tagWidth = MEM_ADDR_BITS - log2Up(sets) + val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, dataLeaf)) - val writeback = Module(new LLCWriteback(2)) + val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) + val writeback = Module(new LLCWriteback(2, refill_cycles)) val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) @@ -497,11 +497,11 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } val numEntries = numRequests * refillCycles diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 42611bc6..21a987fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,49 +2,56 @@ package uncore import Chisel._ import scala.math._ -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) +case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) + +abstract trait MemoryIFSubBundle extends Bundle { + implicit val conf: MemoryIFConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } -trait HasMemAddr extends Bundle { - val addr = UInt(width = MEM_ADDR_BITS) +trait HasMemData extends MemoryIFSubBundle { + val data = Bits(width = conf.dataBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = MEM_TAG_BITS) +trait HasMemAddr extends MemoryIFSubBundle { + val addr = UInt(width = conf.addrBits) } -class MemReqCmd extends HasMemAddr with HasMemTag { +trait HasMemTag extends MemoryIFSubBundle { + val tag = UInt(width = conf.tagBits) +} + +class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag +class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag -class MemData extends HasMemData +class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData -class ioMem extends Bundle { +class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class ioMemPipe extends Bundle { +class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class ioMemSerialized(w: Int) extends Bundle +class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val wide = new MemIO().flip + val narrow = new MemSerializedIO(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -56,7 +63,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -92,12 +99,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -115,12 +122,12 @@ class MemSerdes(w: Int) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int) extends Bundle { - val narrow = new ioMemSerialized(w).flip - val wide = new ioMem +class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { + val narrow = new MemSerializedIO(w).flip + val wide = new MemIO } -class MemDesser(w: Int) extends Module // test rig side +class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -129,7 +136,7 @@ class MemDesser(w: Int) extends Module // test rig side require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -157,13 +164,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -177,10 +184,92 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) + val dataq = Module(new Queue(new MemResp, conf.dataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemIO + } + + require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) + //require(tlconf.clientXactIdBits <= mifconf.tagBits) + + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + val cnt_max = mifconf.dataBeats + + val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val buf_out = Reg(Bits()) + val tag_out = Reg(Bits()) + val addr_out = Reg(Bits()) + val has_data = Reg(init=Bool(false)) + + val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) + val active_in = Reg(init=Bool(false)) + val buf_in = Reg(Bits()) + val tag_in = Reg(UInt(width = mifconf.tagBits)) + + // Decompose outgoing TL Acquires into MemIO cmd and data + when(!active_out && io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + cnt_out := UInt(0) + buf_out := io.uncached.acquire.bits.payload.data + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + } + when(active_out) { + when(mem_cmd_q.io.enq.fire()) { + cmd_sent_out := Bool(true) + } + when(mem_data_q.io.enq.fire()) { + cnt_out := cnt_out + UInt(1) + buf_out := buf_out >> UInt(mifconf.dataBits) + } + when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + active_out := Bool(false) + } + } + + io.uncached.acquire.ready := !active_out + mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) + mem_data_q.io.enq.bits.data := buf_out + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + + // Aggregate incoming MemIO responses into TL Grants + io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) + io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) + io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + when(!active_in && io.mem.resp.valid) { + active_in := Bool(true) + cnt_in := UInt(1) + buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + tag_in := io.mem.resp.bits.tag + } + when(active_in) { + when(io.uncached.grant.fire()) { + active_in := Bool(false) + } + when(io.mem.resp.fire()) { + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + cnt_in := cnt_in + UInt(1) + } + } +} + diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 613a3b65..6889bd7f 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,102 +1,5 @@ package uncore import Chisel._ -import scala.collection.mutable.Stack - -class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { - val meta = Decoupled(mType) - val data = Decoupled(dType) - override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { - val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip - val out = new PairedDataIO(mType, dType) - val meta_chosen = UInt(OUTPUT, log2Up(n)) - val data_chosen = UInt(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { - require(isPow2(count)) - val io = new PairedArbiterIO(mType, dType, n) - val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(init=UInt(0, width = log2Up(count))) - val cnt_next = cnt + UInt(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UInt(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(init=Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} -} - -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new PairedCrossbarIO(mType, dType) - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) @@ -147,7 +50,13 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UInt(width = conf.idBits) } -object FIFOedLogicalNetworkIOWrapper { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } +} + +object DecoupledLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid @@ -159,7 +68,7 @@ object FIFOedLogicalNetworkIOWrapper { } } -object FIFOedLogicalNetworkIOUnwrapper { +object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid @@ -168,9 +77,3 @@ object FIFOedLogicalNetworkIOUnwrapper { out } } - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = dType.clone - override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } -} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index cd50213f..6f877856 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -35,18 +35,24 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.data := Bits(0) acq.write_mask := Bits(0) acq.subword_addr := Bits(0) acq.atomic_opcode := Bits(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) + acq.data := data + acq + } + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { - val acq = apply(a_type, addr, client_xact_id) + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq @@ -57,15 +63,17 @@ object Acquire acq } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData - object Probe { def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -76,16 +84,20 @@ object Probe prb } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasPhysicalAddress + with HasMasterTransactionId { val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr + rel.data := data rel } def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -94,16 +106,45 @@ object Release rel.addr := addr rel.client_xact_id := client_xact_id rel.master_xact_id := master_xact_id + rel.data := UInt(0) + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + val rel = apply(r_type, addr, client_xact_id, master_xact_id) + rel.data := data rel } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasMasterTransactionId + with HasTileLinkData { val r_type = UInt(width = conf.co.releaseTypeWidth) } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Grant +{ + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val gnt = new Grant + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := UInt(0) + gnt + } + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + val gnt = apply(g_type, client_xact_id, master_xact_id) + gnt.data := data + gnt + } +} -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasTileLinkData + with HasClientTransactionId + with HasMasterTransactionId { val g_type = UInt(width = conf.co.grantTypeWidth) } @@ -112,7 +153,7 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -120,7 +161,7 @@ class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val release = new DecoupledIO(new LogicalNetworkIO(new Release)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -129,32 +170,28 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId - type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def masterSourcedClientXactId(in: MasterSourcedWithId): Bits def arbIdx(in: MasterSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] - (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], - out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hookupClientSource[M <: ClientSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) - val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, - arbN, REFILL_CYCLES, hasData _)) + val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload := req.meta.bits.payload - arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready }} } def hookupMasterSource[M <: MasterSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) for (i <- 0 until arbN) { ins(i).valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 67dfde85..3e162583 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -31,16 +31,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.meta.ready - t.acquire.meta.bits := acquire.meta.bits - t.acquire.meta.valid := alloc_arb.io.in(i).ready - - t.acquire.data.bits := acquire.data.bits - t.acquire.data.valid := acquire.data.valid + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires - acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -49,21 +45,18 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Handle releases, which might be voluntary and might have data val release = io.client.release - val voluntary = co.isVoluntary(release.meta.bits.payload) + val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases - t.release.data.bits := release.data.bits - t.release.data.valid := release.data.valid + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases - release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) @@ -83,7 +76,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -92,63 +85,56 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(init=Bool(false)) - val mem_cmd_sent = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := cmd_to_write - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) // DNC + io.client.release.ready := Bool(false) io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.payload.data := UInt(0) - io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.meta.ready := Bool(true) - when( io.client.release.meta.valid ) { - xact := io.client.release.meta.bits.payload - init_client_id_ := io.client.release.meta.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UInt(0) - mem_cmd_sent := Bool(false) - state := s_mem + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - mem_cmd_sent, - init_client_id_) - } . otherwise { state := s_ack } + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } } is(s_ack) { io.client.grant.valid := Bool(true) @@ -158,172 +144,138 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(init=Bool(false)) - val acquire_data_needs_write = Reg(init=Bool(false)) - val release_data_needs_write = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = Reg(init=Bool(false)) - val r_w_mem_cmd_sent = Reg(init=Bool(false)) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) - io.client.probe.bits.payload.master_xact_id := UInt(trackerId) - io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UInt(0) - io.client.probe.bits.header.src := UInt(bankId) - io.client.grant.bits.payload.data := io.master.grant.bits.payload.data - io.client.grant.bits.payload.g_type := grant_type - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read io.master.grant.ready := io.client.grant.ready + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + switch (state) { is(s_idle) { - io.client.acquire.meta.ready := Bool(true) - when( io.client.acquire.meta.valid ) { - xact := io.client.acquire.meta.bits.payload - init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - mem_cnt := UInt(0) - r_w_mem_cmd_sent := Bool(false) - a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) } } is(s_probe) { - when(probe_flags.orR) { - io.client.probe.valid := Bool(true) - io.client.probe.bits.header.dst := curr_p_id - } + // Generate probes + io.client.probe.valid := probe_flags.orR when(io.client.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } - io.client.release.meta.ready := Bool(true) - when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := s_mem - } - when( co.messageHasData(io.client.release.meta.bits.payload)) { - release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.meta.bits.header.src + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } - is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) - } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.acquire.data, - cmd_to_write, - acquire_data_needs_write, - a_w_mem_cmd_sent, - init_client_id_) - } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) - } . otherwise { - state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_ack) { + is(s_make_grant) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } } } - -abstract trait OuterRequestGenerator { - val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UInt(1) - - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { - val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd - master_acq.data.bits.payload := client_data.bits.payload - when(master_acq.meta.fire()) { - cmd_sent := Bool(true) - } - when (do_write) { - master_acq.meta.valid := !cmd_sent - when (master_acq.meta.ready || cmd_sent) { - master_acq.data.valid := client_data.valid - when(master_acq.data.ready) { - client_data.ready:= Bool(true) - mem_cnt := mem_cnt_next - when(mem_cnt === UInt(REFILL_CYCLES-1)) { - trigger := Bool(false) - } - } - } - } - } - - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { - master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd - when(master_acq.meta.ready) { - trigger := Bool(false) - } - } -} From b1df49ba30a9aed51527bba52afc6e151707ce2d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 1 Apr 2014 17:14:45 -0700 Subject: [PATCH 215/374] removed AddressSpaceConstants, CacheConstants, and TileLinkSizeConstants --- uncore/src/main/scala/consts.scala | 28 --------------------------- uncore/src/main/scala/htif.scala | 10 +++++----- uncore/src/main/scala/memserdes.scala | 5 +++++ uncore/src/main/scala/package.scala | 5 +---- uncore/src/main/scala/tilelink.scala | 10 +++++----- 5 files changed, 16 insertions(+), 42 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index b0cdcdd6..64d6c3a7 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -2,7 +2,6 @@ package uncore package constants import Chisel._ -import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { @@ -43,30 +42,3 @@ trait MemoryOpConstants { def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } - -object AddressConstants extends AddressConstants -trait AddressConstants { - val PADDR_BITS = 32 - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; -} - -object CacheConstants extends CacheConstants -trait CacheConstants { - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkSizeConstants { - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - -trait MemoryInterfaceConstants extends - CacheConstants with - AddressConstants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index c4d0ffb0..9ea26f86 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -75,7 +75,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (rx_count === UInt(short_request_bits/w-1)) { cmd := next_cmd size := rx_shifter_in(15,4) - pos := rx_shifter_in(15,4+OFFSET_BITS-3) + pos := rx_shifter_in(15,4+offsetBits-3) seqno := rx_shifter_in(23,16) addr := rx_shifter_in(63,24) } @@ -95,7 +95,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) - val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), Bool(true))) @@ -157,7 +157,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (state === state_mem_finish && io.mem.grant_ack.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) - addr := addr + UInt(1 << OFFSET_BITS-3) + addr := addr + UInt(1 << offsetBits-3) } when (state === state_tx && tx_done) { when (tx_word_count === tx_size) { @@ -176,7 +176,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_req_data = Cat(packet_ram(idx), mem_req_data) } acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq - val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 21a987fd..dea3e801 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,6 +2,11 @@ package uncore import Chisel._ import scala.math._ +case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { + val ppnBits = paddrBits - pgIdxBits + val vpnBits = vaddrBits - pgIdxBits +} + case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) abstract trait MemoryIFSubBundle extends Bundle { diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 4ac83d0b..501d511d 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,7 +1,4 @@ -package object uncore extends - uncore.constants.MemoryOpConstants with - uncore.constants.TileLinkSizeConstants with - uncore.constants.MemoryInterfaceConstants +package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6f877856..451c5844 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration @@ -9,7 +9,7 @@ abstract trait TileLinkSubBundle extends Bundle { } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -69,9 +69,9 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes with HasClientTransactionId with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + val write_mask = Bits(width = conf.writeMaskBits) + val subword_addr = Bits(width = conf.wordAddrBits) + val atomic_opcode = Bits(width = conf.atomicOpBits) } object Probe From bf2ff7804eda45a6a690a32a11c39cafe9481539 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 17 Apr 2014 17:01:40 -0700 Subject: [PATCH 216/374] Add chisel-dependent.sbt for -DchiselVersion="latest.release" If -DchiselVersion is specified on the command line, add the appropriate chisel library to libraryDependencies. --- uncore/chisel-dependent.sbt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uncore/chisel-dependent.sbt diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt new file mode 100644 index 00000000..fa8d5490 --- /dev/null +++ b/uncore/chisel-dependent.sbt @@ -0,0 +1,6 @@ +// Provide a managed dependency on chisel if -DchiselVersion="" is +// supplied on the command line. + +val chiselVersion = System.getProperty("chiselVersion", "None") + +libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From 5613dc7d1ba5278cc9ee00e59242f2624c9fea5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Apr 2014 18:14:12 -0700 Subject: [PATCH 217/374] replaced Lists with Vecs --- uncore/src/main/scala/coherence.scala | 107 ++++++++++++-------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c9d0f4eb..327e31e5 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -52,8 +52,6 @@ abstract class CoherencePolicy { def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - - def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -96,10 +94,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List() - val hasDataAcquireTypeList = List(acquireWriteback) - val hasDataReleaseTypeList = List(acquireWriteback) - val hasDataGrantTypeList = List(grantData) + val hasDataAcquireTypeVec = Vec(acquireWriteback) + val hasDataReleaseTypeVec = Vec(acquireWriteback) + val hasDataGrantTypeVec = Vec(grantData) def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) def isValid (state: UInt): Bool = state != tileInvalid @@ -135,13 +132,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = Bool(false) } class MICoherence extends CoherencePolicyWithUncached { @@ -160,10 +157,10 @@ class MICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -231,15 +228,15 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -302,10 +299,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -386,15 +383,15 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -460,10 +457,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), @@ -550,15 +547,15 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -621,10 +618,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), @@ -711,15 +708,15 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -785,13 +782,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -803,8 +800,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), + M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) )) } def needsWriteback (state: UInt): Bool = { @@ -875,7 +872,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -890,15 +887,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) - } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) From 39681303b83e275c6bfcf22146e6a98720d1920a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Apr 2014 16:55:35 -0700 Subject: [PATCH 218/374] beginning of l2 cache --- uncore/src/main/scala/cache.scala | 431 ++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 uncore/src/main/scala/cache.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala new file mode 100644 index 00000000..a4c2755c --- /dev/null +++ b/uncore/src/main/scala/cache.scala @@ -0,0 +1,431 @@ +package uncore +import Chisel._ + +abstract class CacheConfig { + def sets: Int + def ways: Int + def tl: TileLinkConfiguration + def as: AddressSpaceConfiguration + def dm: Boolean + def states: Int + def lines: Int + def tagbits: Int + def idxbits: Int + def offbits: Int + def untagbits: Int + def rowbits: Int + def metabits: Int + def statebits: Int +} + +case class L2CacheConfig(val sets: Int, val ways: Int, + nReleaseTransactions: Int, nAcquireTransactions: Int, + nrpq: Int, nsdq: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) extends CacheConfig { + def databits = tl.dataBits + def states = tl.co.nMasterStates + def lines = sets*ways + def dm = ways == 1 + def offbits = 0 + def lineaddrbits = tl.addrBits + def idxbits = log2Up(sets) + def waybits = log2Up(ways) + def untagbits = offbits + idxbits + def tagbits = lineaddrbits - idxbits + def databytes = databits/8 + def wordoffbits = log2Up(databytes) + def rowbits = tl.dataBits + def rowbytes = rowbits/8 + def rowoffbits = log2Up(rowbytes) + def refillcycles = tl.dataBits/(rowbits) + def statebits = log2Up(states) + def metabits = statebits + tagbits + + require(states > 0) + require(isPow2(sets)) + require(isPow2(ways)) // TODO: relax this + require(rowbits == tl.dataBits) //TODO: relax this? +} + +abstract trait CacheBundle extends Bundle { + implicit val conf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract trait L2CacheBundle extends Bundle { + implicit val conf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract class ReplacementPolicy { + def way: UInt + def miss: Unit + def hit: Unit +} + +class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { + private val replace = Bool() + replace := Bool(false) + val lfsr = LFSR16(replace) + + def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def miss = replace := Bool(true) + def hit = {} +} + +object L2MetaData { + def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + val meta = new L2MetaData + meta.state := state + meta.tag := tag + meta + } +} +class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { + val state = UInt(width = conf.statebits) + val tag = Bits(width = conf.tagbits) +} + +class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { + val idx = Bits(width = conf.idxbits) +} + +class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { + val way_en = Bits(width = conf.ways) + val data = new L2MetaData() +} + +class L2MetaDataArray(implicit conf: CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + } + + val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) + val rst = rst_cnt < UInt(conf.sets) + when (rst) { rst_cnt := rst_cnt+UInt(1) } + + val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + + when (rst || io.write.valid) { + val addr = Mux(rst, rst_cnt, io.write.bits.idx) + val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val mask = Mux(rst, SInt(-1), io.write.bits.way_en) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + } + val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + + for (w <- 0 until conf.ways) { + val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) + io.resp(w).state := m >> UInt(conf.tagbits) + io.resp(w).tag := m + } + + io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM + io.write.ready := !rst +} + +class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = conf.ways) + val addr = Bits(width = conf.untagbits) +} + +class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { + val wmask = Bits(width = conf.tl.writeMaskBits) + val data = Bits(width = conf.rowbits) +} + +class L2DataArray(implicit conf: L2CacheConfig) extends Module { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + } + + val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) + val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) + + val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + for (w <- 0 until conf.ways) { + val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) +{ + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.client.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe request generation + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + + // Handle releases, which might be voluntary and might have data + val release = io.client.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + + // Free finished transactions + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) + ack.ready := Bool(true) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out +} + + +abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO + val tile_incoherent = Bits(INPUT, ln.nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) + + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) + + switch (state) { + is(s_idle) { + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + } + } + is(s_mem) { + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + //TODO: Will need id reg for merged release xacts + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + + val probe_initial_flags = Bits(width = ln.nClients) + probe_initial_flags := Bits(0) + if (ln.nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read + io.master.grant.ready := io.client.grant.ready + + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + + switch (state) { + is(s_idle) { + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + probe_flags := probe_initial_flags + if(ln.nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + is(s_probe) { + // Generate probes + io.client.probe.valid := probe_flags.orR + when(io.client.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From f8f29c69b8ad5c619cbe849c69f0eda690ba70dc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 23 Apr 2014 16:24:20 -0700 Subject: [PATCH 219/374] MetaData & friends moved to uncore/ --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a4c2755c..bd677c08 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -74,34 +74,34 @@ class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolic def hit = {} } -object L2MetaData { +object MetaData { def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { - val meta = new L2MetaData + val meta = new MetaData meta.state := state meta.tag := tag meta } } -class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { +class MetaData(implicit val conf: CacheConfig) extends CacheBundle { val state = UInt(width = conf.statebits) val tag = Bits(width = conf.tagbits) } -class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { +class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { val idx = Bits(width = conf.idxbits) } -class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { +class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { val way_en = Bits(width = conf.ways) - val data = new L2MetaData() + val data = new MetaData() } -class L2MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + val read = Decoupled(new MetaReadReq).flip + val write = Decoupled(new MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new MetaData).asOutput} } val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) From 3f53d532c25c2039735a3ee2f86f263ce54b4eb1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 14:58:38 -0700 Subject: [PATCH 220/374] uniquify tilelink conf val name for easier subtyping --- uncore/src/main/scala/tilelink.scala | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 451c5844..22941d29 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,24 +4,24 @@ import Chisel._ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { - implicit val conf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val tlconf: TileLinkConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = tlconf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = conf.clientXactIdBits) + val client_xact_id = Bits(width = tlconf.clientXactIdBits) } trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = conf.masterXactIdBits) + val master_xact_id = Bits(width = tlconf.masterXactIdBits) } trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = conf.dataBits) + val data = Bits(width = tlconf.dataBits) } trait SourcedMessage extends Bundle @@ -64,14 +64,14 @@ object Acquire } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = conf.writeMaskBits) - val subword_addr = Bits(width = conf.wordAddrBits) - val atomic_opcode = Bits(width = conf.atomicOpBits) + val a_type = UInt(width = tlconf.co.acquireTypeWidth) + val write_mask = Bits(width = tlconf.writeMaskBits) + val subword_addr = Bits(width = tlconf.wordAddrBits) + val atomic_opcode = Bits(width = tlconf.atomicOpBits) } object Probe @@ -85,10 +85,10 @@ object Probe } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = conf.co.probeTypeWidth) + val p_type = UInt(width = tlconf.co.probeTypeWidth) } object Release @@ -116,12 +116,12 @@ object Release } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = tlconf.co.releaseTypeWidth) } object Grant @@ -141,14 +141,14 @@ object Grant } } -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = conf.co.grantTypeWidth) + val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { From 1163131d1e44804e065a8edea3065111745f535f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 15:17:05 -0700 Subject: [PATCH 221/374] TileLinkIO.GrantAck -> TileLinkIO.Finish --- uncore/src/main/scala/cache.scala | 10 +++++----- uncore/src/main/scala/htif.scala | 8 ++++---- uncore/src/main/scala/tilelink.scala | 16 ++++++++-------- uncore/src/main/scala/uncore.scala | 10 +++++----- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bd677c08..3af6f29f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,9 +214,9 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -239,7 +239,7 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -423,7 +423,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 9ea26f86..41d6df28 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -154,7 +154,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } mem_acked := Bool(false) } - when (state === state_mem_finish && io.mem.grant_ack.ready) { + when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -186,9 +186,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module - io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid - io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 22941d29..68da8213 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -148,14 +148,14 @@ class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMes val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } @@ -214,9 +214,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupMasterSource(io.in.map(_.grant), io.out.grant) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { @@ -233,9 +233,9 @@ abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) e io.in.map{ _.probe.bits := io.out.probe.bits } io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract trait AppendsArbiterId { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3e162583..0a3b7d6e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -64,9 +64,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -89,7 +89,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -273,7 +273,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 52c6de56414b9da67957deb258d7bc0bcb3f41c5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 19:11:36 -0700 Subject: [PATCH 222/374] DRAMSideLLCLike trait. TSHRFile. New L2 config objects. --- uncore/src/main/scala/cache.scala | 114 ++++++++++++++++++++--------- uncore/src/main/scala/llc.scala | 14 ++-- uncore/src/main/scala/uncore.scala | 31 +++++--- 3 files changed, 106 insertions(+), 53 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3af6f29f..3d42e97d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CacheConfig { +trait CacheConfig { def sets: Int def ways: Int def tl: TileLinkConfiguration @@ -18,12 +18,16 @@ abstract class CacheConfig { def statebits: Int } -case class L2CacheConfig(val sets: Int, val ways: Int, - nReleaseTransactions: Int, nAcquireTransactions: Int, - nrpq: Int, nsdq: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) extends CacheConfig { - def databits = tl.dataBits +case class L2CacheConfig( + val sets: Int, val ways: Int, + val nrpq: Int, val nsdq: Int, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) + extends CoherenceAgentConfiguration + with CacheConfig +{ def states = tl.co.nMasterStates def lines = sets*ways def dm = ways == 1 @@ -33,8 +37,9 @@ case class L2CacheConfig(val sets: Int, val ways: Int, def waybits = log2Up(ways) def untagbits = offbits + idxbits def tagbits = lineaddrbits - idxbits - def databytes = databits/8 - def wordoffbits = log2Up(databytes) + def wordbits = 64 + def wordbytes = wordbits/8 + def wordoffbits = log2Up(wordbytes) def rowbits = tl.dataBits def rowbytes = rowbits/8 def rowoffbits = log2Up(rowbytes) @@ -45,17 +50,17 @@ case class L2CacheConfig(val sets: Int, val ways: Int, require(states > 0) require(isPow2(sets)) require(isPow2(ways)) // TODO: relax this - require(rowbits == tl.dataBits) //TODO: relax this? + require(refillcycles == 1) //TODO: relax this? } abstract trait CacheBundle extends Bundle { - implicit val conf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val cacheconf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] } abstract trait L2CacheBundle extends Bundle { - implicit val conf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val l2cacheconf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } abstract class ReplacementPolicy { @@ -64,31 +69,31 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) def miss = replace := Bool(true) def hit = {} } object MetaData { - def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { val meta = new MetaData meta.state := state meta.tag := tag meta } } -class MetaData(implicit val conf: CacheConfig) extends CacheBundle { - val state = UInt(width = conf.statebits) - val tag = Bits(width = conf.tagbits) +class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { + val state = UInt(width = cacheconf.statebits) + val tag = Bits(width = cacheconf.tagbits) } -class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { - val idx = Bits(width = conf.idxbits) +class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { + val idx = Bits(width = cacheconf.idxbits) } class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { @@ -128,27 +133,26 @@ class MetaDataArray(implicit conf: CacheConfig) extends Module { io.write.ready := !rst } -class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { - val way_en = Bits(width = conf.ways) - val addr = Bits(width = conf.untagbits) +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = l2cacheconf.ways) + val addr = Bits(width = l2cacheconf.tl.addrBits) } class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.rowbits) + val data = Bits(width = conf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} } - val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) - val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) - - val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) for (w <- 0 until conf.ways) { val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { @@ -161,11 +165,51 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) -{ +trait InternalRequestState extends CacheBundle { + val tag_match = Bool() + val old_meta = new MetaData + val way_en = Bits(width = cacheconf.ways) +} + +class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) + with InternalRequestState + +class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) + with InternalRequestState + +class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) + val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) - // Create SHRs for outstanding transactions + val tshrfile = Module(new TSHRFile(bankId)) + + io.client <> tshrfile.io.client + io.master <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val meta_read_req = Decoupled(new MetaReadReq) + val meta_write_req = Decoupled(new MetaWriteReq) + val data_read_req = Decoupled(new MetaReadReq) + val data_write_req = Decoupled(new MetaWriteReq) + } + + // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) @@ -222,7 +266,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + io.outer <> outer_arb.io.out } diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1bfa4da7..382a5f96 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -343,13 +343,15 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module -{ +abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } +} +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +{ val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -497,12 +499,8 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module -{ - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) + extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0a3b7d6e..cbcd39b6 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,18 +1,27 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { +trait CoherenceAgentConfiguration { + def tl: TileLinkConfiguration + def nReleaseTransactions: Int + def nAcquireTransactions: Int +} + +abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO - val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput + val client = (new TileLinkIO()(conf.tl)).flip + val master = new UncachedTileLinkIO()(conf.tl) + val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) +case class L2CoherenceAgentConfiguration( + val tl: TileLinkConfiguration, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int) extends CoherenceAgentConfiguration -class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) -{ +class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) // Create SHRs for outstanding transactions @@ -76,7 +85,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { +abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -94,7 +103,8 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -143,7 +153,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } From 023722992162c123fee90d2395c14d7c866a2dd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 29 Apr 2014 16:49:18 -0700 Subject: [PATCH 223/374] client/master -> inner/outer --- uncore/src/main/scala/cache.scala | 160 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 160 ++++++++++++++--------------- 2 files changed, 160 insertions(+), 160 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3d42e97d..4eb28f97 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -191,8 +191,8 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) - io.client <> tshrfile.io.client - io.master <> tshrfile.io.outer + tshrfile.io.inner <> io.inner + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -200,7 +200,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new InternalTileLinkIO).flip + val inner = (new InternalTileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) @@ -218,27 +218,27 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -246,26 +246,26 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } @@ -273,18 +273,18 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -293,46 +293,46 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -361,7 +361,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -369,37 +369,37 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -415,18 +415,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -434,7 +434,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -444,30 +444,30 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cbcd39b6..c8e33216 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -9,8 +9,8 @@ trait CoherenceAgentConfiguration { abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO()(conf.tl)).flip - val master = new UncachedTileLinkIO()(conf.tl) + val inner = (new TileLinkIO()(conf.tl)).flip + val outer = new UncachedTileLinkIO()(conf.tl) val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } @@ -33,27 +33,27 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -61,45 +61,45 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out } abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -109,46 +109,46 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -178,7 +178,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -186,37 +186,37 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -232,18 +232,18 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -251,7 +251,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -261,30 +261,30 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 45172f1f37eb5e0633bf9eb2cd700d3cd15979f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 May 2014 01:44:59 -0700 Subject: [PATCH 224/374] parameterize metadataarray --- uncore/src/main/scala/cache.scala | 91 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4eb28f97..610317d2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -14,8 +14,6 @@ trait CacheConfig { def offbits: Int def untagbits: Int def rowbits: Int - def metabits: Int - def statebits: Int } case class L2CacheConfig( @@ -45,7 +43,6 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) - def metabits = statebits + tagbits require(states > 0) require(isPow2(sets)) @@ -79,54 +76,61 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -object MetaData { - def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { - val meta = new MetaData - meta.state := state - meta.tag := tag - meta - } -} -class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { - val state = UInt(width = cacheconf.statebits) +abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) } +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { +class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) - val data = new MetaData() + val data = gen.clone + override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl + def gen = resetMeta.clone val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new MetaData).asOutput} + val write = Decoupled(new MetaWriteReq(gen)).flip + val resp = Vec.fill(conf.ways){gen.asOutput} } + val metabits = resetMeta.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val data = Mux(rst, resetMeta, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) - io.resp(w).state := m >> UInt(conf.tagbits) - io.resp(w).tag := m + val m = tag(metabits*(w+1)-1, metabits*w) + io.resp(w) := gen.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -165,23 +169,23 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -trait InternalRequestState extends CacheBundle { +trait L2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new MetaData - val way_en = Bits(width = cacheconf.ways) + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) } -class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) - with InternalRequestState +class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) + with L2InternalRequestState -class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) - with InternalRequestState +class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) + with L2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } @@ -191,6 +195,21 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) + // tags + val meta = Module(new MetaDataArray(new L2MetaData)) + + // data + val data = Module(new L2DataArray) + + // replacement policy + val replacer = new RandomReplacement +/* + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) +*/ + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -204,9 +223,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq) - val data_read_req = Decoupled(new MetaReadReq) - val data_write_req = Decoupled(new MetaWriteReq) + val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) + val data_read_req = Decoupled(new L2DataReadReq) + val data_write_req = Decoupled(new L2DataWriteReq) } // Create TSHRs for outstanding transactions From bc3ef1011eb7182d48f739bfff823772e36d2f62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 May 2014 12:59:45 -0700 Subject: [PATCH 225/374] correct use of function value to initialize MetaDataArray --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 610317d2..e41e1cfb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -104,16 +104,16 @@ class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends Me override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - def gen = resetMeta.clone + val rstVal = gen() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq(gen)).flip - val resp = Vec.fill(conf.ways){gen.asOutput} + val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip + val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = resetMeta.getWidth + val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } @@ -122,7 +122,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, resetMeta, io.write.bits.data).toBits + val data = Mux(rst, rstVal, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } @@ -130,7 +130,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext for (w <- 0 until conf.ways) { val m = tag(metabits*(w+1)-1, metabits*w) - io.resp(w) := gen.fromBits(m) + io.resp(w) := rstVal.clone.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -196,7 +196,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) // tags - val meta = Module(new MetaDataArray(new L2MetaData)) + val meta = Module(new MetaDataArray(() => new L2MetaData)) // data val data = Module(new L2DataArray) From 0e39346a126778862361b8adf43834f14e0fd3fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 May 2014 01:51:46 -0700 Subject: [PATCH 226/374] L2-specific metadataarray wrapper, hookups to tshrfile --- uncore/src/main/scala/cache.scala | 212 +++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e41e1cfb..e5851a6b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -43,6 +43,7 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) + def nTSHRs = nReleaseTransactions + nAcquireTransactions require(states > 0) require(isPow2(sets)) @@ -60,6 +61,16 @@ abstract trait L2CacheBundle extends Bundle { override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } +trait HasL2Id extends L2CacheBundle { + val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) +} + +trait HasL2InternalRequestState extends L2CacheBundle { + val tag_match = Bool() + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -80,20 +91,6 @@ abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle val tag = Bits(width = cacheconf.tagbits) } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData - with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) -} - -/* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) -} -*/ - class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } @@ -137,49 +134,125 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { +object L2MetaData { + def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { + val meta = new L2MetaData + meta.tag := tag + meta.state := state + meta.sharers := sharers + meta + } +} +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + +class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq + with HasL2Id { + val tag = Bits(width = l2cacheconf.tagbits) +} + +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) + with HasL2Id + +class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) + val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + + val replacer = new RandomReplacement + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.old_meta := Mux(s2_tag_match, + L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { val way_en = Bits(width = l2cacheconf.ways) val addr = Bits(width = l2cacheconf.tl.addrBits) } -class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { - val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.tl.dataBits) +class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { + val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) + val data = Bits(width = l2cacheconf.tl.dataBits) +} + +class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { + val data = Bits(width = l2cacheconf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} + val resp = Valid(new L2DataResp) } val waddr = io.write.bits.addr val raddr = io.read.bits.addr val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - for (w <- 0 until conf.ways) { + val resp = (0 until conf.ways).map { w => val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } - io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) io.read.ready := Bool(true) io.write.ready := Bool(true) } -trait L2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2MetaData - val way_en = Bits(width = l2cacheconf.ways) -} - class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) @@ -194,23 +267,16 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - - // tags - val meta = Module(new MetaDataArray(() => new L2MetaData)) - - // data + val meta = Module(new L2MetaDataArray) val data = Module(new L2DataArray) - // replacement policy - val replacer = new RandomReplacement -/* - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) -*/ - tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -219,19 +285,31 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput - val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) - val data_read_req = Decoupled(new L2DataReadReq) - val data_write_req = Decoupled(new L2DataWriteReq) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, i) => a <> i } } // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + val trackerList = (0 until conf.nReleaseTransactions).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId)) + } ++ (conf.nReleaseTransactions until nTrackers).map { id => + Module(new L2AcquireTracker(id, bankId)) + } // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -251,10 +329,11 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + //io.inner.probe <> probe_arb.io.out + //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -272,9 +351,10 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + //io.inner.grant <> grant_arb.io.out + //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -282,21 +362,35 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) - // Create an arbiter for the one memory port + // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + + } abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip } val c_acq = io.inner.acquire.bits From 364a6de2145dd3e81cd432cf31fa56bb177eef3f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 18 May 2014 19:26:35 -0700 Subject: [PATCH 227/374] Use Mem instead of Vec[Reg] --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 41d6df28..cc1c1c62 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -84,7 +84,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR val packet_ram_depth = long_request_bits/short_request_bits-1 - val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) when (rx_word_done && io.host.in.ready) { packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in } From 3c329df7e761295b4d8e1b8b7bd3a41b32630271 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 28 May 2014 13:35:08 -0700 Subject: [PATCH 228/374] refactor Metadata, clean and expand coherence API --- uncore/src/main/scala/cache.scala | 102 ++-- uncore/src/main/scala/coherence.scala | 807 ++++++++++++++------------ uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 6 +- 4 files changed, 502 insertions(+), 415 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e5851a6b..9a5afee3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -67,7 +67,7 @@ trait HasL2Id extends L2CacheBundle { trait HasL2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new L2MetaData + val old_meta = new L2Metadata val way_en = Bits(width = l2cacheconf.ways) } @@ -87,46 +87,45 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { +abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) + val coh: CoherenceMetadata } class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { +class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) val data = gen.clone override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - val rstVal = gen() + val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) + val waddr = Mux(rst, rst_cnt, io.write.bits.idx) + val wdata = Mux(rst, rstVal, io.write.bits.data).toBits + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) - + val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { - val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, rstVal, io.write.bits.data).toBits - val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) + tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) } - val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(metabits*(w+1)-1, metabits*w) + val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -134,26 +133,23 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -object L2MetaData { - def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { - val meta = new L2MetaData +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + val meta = new L2Metadata meta.tag := tag - meta.state := state - meta.sharers := sharers + meta.coh := coh meta } } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData +class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) + val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone } /* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) +class L3Metadata(implicit conf: L3CacheConfig) extends Metadata + with L3CacheBundle { + val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? } */ @@ -162,14 +158,14 @@ class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq val tag = Bits(width = l2cacheconf.tagbits) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { +class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip @@ -177,7 +173,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val resp = Valid(new L2MetaResp) } - val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -186,11 +182,11 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR - val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) - val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = new RandomReplacement val s1_replaced_way_en = UIntToOH(replacer.way) @@ -202,7 +198,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match io.resp.bits.old_meta := Mux(s2_tag_match, - L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } @@ -267,7 +263,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetaDataArray) + val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner @@ -300,7 +296,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, i) => a <> i } + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } } // Create TSHRs for outstanding transactions @@ -331,9 +332,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Handle probe requests doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - //io.inner.probe <> probe_arb.io.out - //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -352,9 +350,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - //io.inner.grant <> grant_arb.io.out - //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -372,7 +367,8 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) } @@ -406,6 +402,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -430,15 +427,28 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca xact.client_xact_id, UInt(trackerId)) + switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + state := s_mem } } +/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits + old_meta := io.meta.resp.bits.old_meta + state := Mux(s_meta_write +Mux(co.messageHasData(xact), s_mem, s_ack) + } + */ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -463,7 +473,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -491,7 +501,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -511,7 +521,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 327e31e5..45a1970c 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,7 +1,86 @@ package uncore import Chisel._ -abstract class CoherencePolicy { +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + var res = default; + for ((t, v) <- mapping.reverse){ + res = Mux(t, v, res); + } + res + } +} + +abstract class CoherenceMetadata extends Bundle + +object ClientMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy) = { + val m = new ClientMetadata + m.state := state + m + } +} +class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata()(c).asInstanceOf[this.type] +} + +object MasterMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + val m = new MasterMetadata + m.state := state + m.sharers := m.sharers.flush() + m + } + def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + val m = apply(state) + m.sharers := sharers + m + } +} +class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.masterStateWidth) + val sharers = c.dir.clone + override def clone = new MasterMetadata()(c).asInstanceOf[this.type] +} +/* +class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { + val cstate = UInt(width = outer.clientStateWidth) + val mstate = UInt(width = inner.masterStateWidth) + val sharers = inner.dir.sharers.clone +} +*/ + +abstract class DirectoryRepresentation extends Bundle { + val sharers: UInt + def pop(id: UInt): DirectoryRepresentation + def push(id: UInt): DirectoryRepresentation + def flush(dummy: Int = 0): DirectoryRepresentation + def none(dummy: Int = 0): Bool + def one(dummy: Int = 0): Bool +} + +class NullRepresentation extends DirectoryRepresentation { + val sharers = UInt(0) + def pop(id: UInt) = this + def push(id: UInt) = this + def flush(dummy: Int = 0) = this + def none(dummy: Int = 0) = Bool(false) + def one(dummy: Int = 0) = Bool(false) +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation { + val sharers = UInt(width = nClients) + def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } + def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } + def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = sharers === UInt(0) + def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +} + +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -15,26 +94,29 @@ abstract class CoherencePolicy { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, state: UInt): Bool - def isValid (state: UInt): Bool + def isHit (cmd: UInt, m: ClientMetadata): Bool + def isValid (m: ClientMetadata): Bool + def isHit (incoming: Acquire, m: MasterMetadata): Bool + def isValid (m: MasterMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool - def needsWriteback (state: UInt): Bool + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool + def needsWriteback(m: ClientMetadata): Bool - def newStateOnHit(cmd: UInt, state: UInt): UInt - def newStateOnCacheControl(cmd: UInt): UInt - def newStateOnWriteback(): UInt - def newStateOnFlush(): UInt - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): UInt + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata + def clientMetadataOnFlush: ClientMetadata + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata + def masterMetadataOnFlush: MasterMetadata + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, global_state: UInt): UInt + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt def getGrantType(a_type: UInt, count: UInt): UInt def getGrantType(rel: Release, count: UInt): UInt @@ -45,9 +127,10 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UInt, global_state: UInt): Bool - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool + def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -63,85 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions +abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) + with UncachedTransactions -abstract class IncoherentPolicy extends CoherencePolicy { - // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): UInt = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) - def getGrantType(rel: Release, count: UInt): UInt = UInt(0) - def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) - def requiresAckForGrant(g_type: UInt) = Bool(true) - def requiresAckForRelease(r_type: UInt) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) - -} - -class ThreeStateIncoherence extends IncoherentPolicy { - def nClientStates = 3 - def nMasterStates = 0 - def nAcquireTypes = 3 - def nProbeTypes = 0 - def nReleaseTypes = 2 - def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataAcquireTypeVec = Vec(acquireWriteback) - val hasDataReleaseTypeVec = Vec(acquireWriteback) - val hasDataGrantTypeVec = Vec(grantData) - - def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UInt): Bool = state != tileInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty - def needsWriteback (state: UInt): Bool = state === tileDirty - - def newState(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) - } - def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) - def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), - grantAck -> tileInvalid - )) - } - - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData - - def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) - case rel: Release => Bool(false) - case _ => Bool(false) - } - def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = Bool(false) -} - -class MICoherence extends CoherencePolicyWithUncached { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -149,8 +157,8 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -162,43 +170,54 @@ class MICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) + def isValid (m: ClientMetadata): Bool = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) + def isValid (m: MasterMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileValid), Array( - M_INV -> (state === tileValid), - M_CLN -> (state === tileValid) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientValid), Array( + M_INV -> (m.state === clientValid), + M_CLN -> (m.state === clientValid) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = state - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileValid - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> tileValid, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientValid + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> clientValid, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseCopyAck) -> m )) } @@ -211,11 +230,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -224,7 +243,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -257,7 +276,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -268,22 +287,21 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence extends CoherencePolicyWithUncached { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -291,8 +309,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -304,50 +322,64 @@ class MEICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) + def isValid (m: ClientMetadata) = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileExclusiveClean, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientExclusiveClean + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -360,15 +392,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -379,7 +411,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -413,7 +445,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -425,15 +457,14 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -441,7 +472,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence extends CoherencePolicyWithUncached { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -449,8 +480,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -462,56 +493,70 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> tileExclusiveDirty, - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> clientExclusiveDirty, + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -524,15 +569,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -543,7 +588,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -576,8 +621,7 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -586,15 +630,14 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -602,7 +645,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence extends CoherencePolicyWithUncached { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -610,8 +653,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -623,56 +666,71 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -685,15 +743,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -704,7 +762,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -738,7 +796,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -750,15 +808,14 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -767,15 +824,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence extends CoherencePolicyWithUncached { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 0 + def nMasterStates = 3 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -787,72 +845,92 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), - M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), + M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( - tileExclusiveClean -> tileExclusiveDirty, - tileMigratoryClean -> tileMigratoryDirty)), state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( - acquireReadExclusive -> tileExclusiveDirty, - acquireReadShared -> tileExclusiveClean)), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( - acquireInvalidateOthers -> tileMigratoryDirty, - acquireReadExclusive -> tileMigratoryDirty, - acquireReadShared -> tileMigratoryClean)) - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeInvalidateOthers -> tileInvalid, - probeCopy -> state, - probeDowngrade -> MuxLookup(state, tileShared, Array( - tileExclusiveClean -> tileSharedByTwo, - tileExclusiveDirty -> tileSharedByTwo, - tileSharedByTwo -> tileShared, - tileMigratoryClean -> tileSharedByTwo, - tileMigratoryDirty -> tileInvalid)) + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( + Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> m.state, + probeDowngrade -> MuxLookup(m.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid)) + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), + popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m, + is(releaseDowngradeDataMigratory) -> m, + is(releaseDowngradeAckHasCopy) -> m, + is(releaseInvalidateDataMigratory) -> next, + is(releaseInvalidateAckMigratory) -> next )) } + def getUncachedReadAcquireType = acquireReadUncached def getUncachedWriteAcquireType = acquireWriteUncached def getUncachedReadWordAcquireType = acquireReadWordUncached @@ -862,28 +940,28 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -906,7 +984,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadWordUncached -> grantReadWordUncached, acquireWriteWordUncached -> grantWriteWordUncached, acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } def getGrantType(rel: Release, count: UInt): UInt = { @@ -916,7 +994,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -929,15 +1007,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index dea3e801..22fa933c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -232,7 +232,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c8e33216..09bb0714 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -167,7 +167,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -195,7 +195,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -215,7 +215,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src From 1ae7a9376cb2a0ec631eb1d634171f767327a20c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 13 Jun 2014 03:25:52 -0700 Subject: [PATCH 229/374] Fix unhandled LLC writeback hazard --- uncore/src/main/scala/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..8da87042 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -438,10 +438,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From a04ef4f5f42e8f5ed758d47c89656266b9ac12ad Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 11:44:43 -0700 Subject: [PATCH 230/374] Quick change to work with new Width class. Replace .width with .needWidth() --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..28cb5217 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 8da87042..1c8671d0 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 + val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From 0020ded36745bad05fa7376d08b828311f84e6cb Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 14:53:48 -0700 Subject: [PATCH 231/374] Replace needWidth() with getWidth. --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 28cb5217..29eba5da 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1c8671d0..3a21cdca 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 - val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From f411fdcce39c354061348bd74968e86eaaeee100 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 8 Aug 2014 12:21:57 -0700 Subject: [PATCH 232/374] Full conversion to params. Compiles but does not elaborate. --- uncore/src/main/scala/cache.scala | 252 +++++++++----------------- uncore/src/main/scala/htif.scala | 54 +++--- uncore/src/main/scala/llc.scala | 37 ++-- uncore/src/main/scala/memserdes.scala | 94 +++++----- uncore/src/main/scala/network.scala | 57 +++--- uncore/src/main/scala/tilelink.scala | 106 ++++++----- uncore/src/main/scala/uncore.scala | 71 ++++---- 7 files changed, 301 insertions(+), 370 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9a5afee3..247110a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,75 +1,18 @@ package uncore import Chisel._ -trait CacheConfig { - def sets: Int - def ways: Int - def tl: TileLinkConfiguration - def as: AddressSpaceConfiguration - def dm: Boolean - def states: Int - def lines: Int - def tagbits: Int - def idxbits: Int - def offbits: Int - def untagbits: Int - def rowbits: Int -} - -case class L2CacheConfig( - val sets: Int, val ways: Int, - val nrpq: Int, val nsdq: Int, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) - extends CoherenceAgentConfiguration - with CacheConfig -{ - def states = tl.co.nMasterStates - def lines = sets*ways - def dm = ways == 1 - def offbits = 0 - def lineaddrbits = tl.addrBits - def idxbits = log2Up(sets) - def waybits = log2Up(ways) - def untagbits = offbits + idxbits - def tagbits = lineaddrbits - idxbits - def wordbits = 64 - def wordbytes = wordbits/8 - def wordoffbits = log2Up(wordbytes) - def rowbits = tl.dataBits - def rowbytes = rowbits/8 - def rowoffbits = log2Up(rowbytes) - def refillcycles = tl.dataBits/(rowbits) - def statebits = log2Up(states) - def nTSHRs = nReleaseTransactions + nAcquireTransactions - - require(states > 0) - require(isPow2(sets)) - require(isPow2(ways)) // TODO: relax this - require(refillcycles == 1) //TODO: relax this? -} - -abstract trait CacheBundle extends Bundle { - implicit val cacheconf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] -} - -abstract trait L2CacheBundle extends Bundle { - implicit val l2cacheconf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] -} - -trait HasL2Id extends L2CacheBundle { - val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) -} - -trait HasL2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2Metadata - val way_en = Bits(width = l2cacheconf.ways) -} +case object NSets extends Field[Int] +case object NWays extends Field[Int] +case object IsDM extends Field[Boolean] +case object TagBits extends Field[Int] +case object IdxBits extends Field[Int] +case object OffBits extends Field[Int] +case object UntagBits extends Field[Int] +case object WayBits extends Field[Int] +case object RowBits extends Field[Int] +case object WordBits extends Field[Int] +case object RefillCycles extends Field[Int] +case object Replacer extends Field[() => ReplacementPolicy] abstract class ReplacementPolicy { def way: UInt @@ -77,54 +20,53 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(ways: Int) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) + def way = if(ways == 1) UInt(0) else lfsr(log2Up(ways)-1,0) def miss = replace := Bool(true) def hit = {} } -abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { - val tag = Bits(width = cacheconf.tagbits) +abstract class Metadata extends Bundle { + val tag = Bits(width = params(TagBits)) val coh: CoherenceMetadata } -class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { - val idx = Bits(width = cacheconf.idxbits) +class MetaReadReq extends Bundle { + val idx = Bits(width = params(IdxBits)) } -class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { - val way_en = Bits(width = conf.ways) +class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { + val way_en = Bits(width = params(WayBits)) val data = gen.clone - override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] + override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { - implicit val tl = conf.tl +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} + val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) - val rst = rst_cnt < UInt(conf.sets) + val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) + val rst = rst_cnt < UInt(params(NSets)) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) + val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until conf.ways) { + for (w <- 0 until params(NWays)) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -133,62 +75,63 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConf io.write.ready := !rst } +trait HasL2Id extends Bundle { + val id = UInt(width = log2Up(params(NTransactors))) +} + +trait HasL2InternalRequestState extends Bundle { + val tag_match = Bool() + val old_meta = new L2Metadata + val way_en = Bits(width = params(NWays)) +} + object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + def apply(tag: Bits, coh: MasterMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata - with L2CacheBundle { - val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone +class L2Metadata extends Metadata { + val coh = params(TLCoherence).masterMetadataOnFlush.clone } -/* -class L3Metadata(implicit conf: L3CacheConfig) extends Metadata - with L3CacheBundle { - val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? -} -*/ - -class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq - with HasL2Id { - val tag = Bits(width = l2cacheconf.tagbits) +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = params(TagBits)) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle +class L2MetaResp extends Bundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { - implicit val tl = conf.tl +class L2MetadataArray extends Module { + val (co, ways) = (params(TLCoherence), params(NWays)) val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip val resp = Valid(new L2MetaResp) } - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - val replacer = new RandomReplacement + val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => @@ -203,23 +146,21 @@ class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val way_en = Bits(width = l2cacheconf.ways) - val addr = Bits(width = l2cacheconf.tl.addrBits) +class L2DataReadReq extends Bundle with HasL2Id { + val way_en = Bits(width = params(NWays)) + val addr = Bits(width = params(TLAddrBits)) } -class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { - val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) } -class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) } -class L2DataArray(implicit conf: L2CacheConfig) extends Module { +class L2DataArray extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -228,9 +169,9 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - val resp = (0 until conf.ways).map { w => - val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) + val resp = (0 until params(NWays)).map { w => + val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -244,23 +185,11 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with HasL2InternalRequestState +class L2HellaCache(bankId: Int) extends CoherenceAgent { -class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with HasL2InternalRequestState - -class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { - implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) -} - -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + require(isPow2(params(NSets))) + require(isPow2(params(NWays))) + require(params(RefillCycles) == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -278,12 +207,12 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA } -class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class TSHRFile(bankId: Int) extends Module { + val (co, nClients) = (params(TLCoherence), params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) val meta_resp = Valid(new L2MetaResp).flip @@ -305,10 +234,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } // Create TSHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map { id => + val trackerList = (0 until params(NReleaseTransactors)).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (conf.nReleaseTransactions until nTrackers).map { id => + } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -373,12 +301,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } -abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class L2XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) val meta_read = Decoupled(new L2MetaReadReq) @@ -397,11 +325,11 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits @@ -460,16 +388,16 @@ Mux(co.messageHasData(xact), s_mem, s_ack) } } -class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -480,12 +408,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -501,7 +429,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -525,9 +453,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -550,7 +478,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -558,7 +486,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..153bad3a 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,6 +4,11 @@ import Chisel._ import Node._ import uncore._ +case object HTIFWidth extends Field[Int] +case object HTIFNSCR extends Field[Int] +case object HTIFOffsetBits extends Field[Int] +case object HTIFNCores extends Field[Int] + class HostIO(val w: Int) extends Bundle { val clk = Bool(OUTPUT) @@ -20,43 +25,48 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO(ntiles: Int) extends Bundle +class HTIFIO extends Bundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(ntiles)) + val id = UInt(INPUT, log2Up(params(HTIFNCores))) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO(n: Int) extends Bundle +class SCRIO extends Bundle { - val rdata = Vec.fill(n){Bits(INPUT, 64)} + val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(n)) + val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(pcr_RESET: Int) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) val io = new Bundle { val host = new HostIO(w) - val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val cpu = Vec.fill(nCores){new HTIFIO().flip} val mem = new TileLinkIO - val scr = new SCRIO(nSCR) + val scr = new SCRIO } + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + conf.dataBits + val long_request_bits = short_request_bits + dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -92,7 +102,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) - val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR @@ -114,13 +124,13 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val mem_acked = Reg(init=Bool(false)) val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_gsrc = Reg(UInt()) val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) @@ -168,8 +178,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } var mem_req_data: Bits = null - for (i <- 0 until conf.dataBits/short_request_bits) { - val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + for (i <- 0 until dataBits/short_request_bits) { + val idx = UInt(i, log2Up(dataBits/short_request_bits)) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } @@ -184,7 +194,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.master_xact_id := mem_gxid @@ -194,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) - for (i <- 0 until nTiles) { + for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) val my_ipi = Reg(init=Bool(false)) @@ -211,7 +221,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } cpu.ipi_rep.valid := my_ipi cpu.ipi_req.ready := Bool(true) - for (j <- 0 until nTiles) { + for (j <- 0 until nCores) { when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { my_ipi := Bool(true) } @@ -239,8 +249,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(0) := UInt(nCores) + scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..5bc89e38 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,26 +72,26 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd +class LLCDataReq(ways: Int) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr +class LLCTagReq(ways: Int) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) + val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) val mem = new MemPipeIO @@ -100,14 +100,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im } class MSHR extends Bundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = params(MIFAddrBits)) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = conf.addrBits - log2Up(sets)) + val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -183,10 +183,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new MemPipeIO } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFC io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = conf.addrBits)) + val writeback = Decoupled(UInt(width = params(MIFAddrBits))) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -300,7 +300,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class MemReqArb(n: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Vec.fill(n){new MemIO().flip} @@ -343,16 +343,16 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { +abstract class DRAMSideLLCLike extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { - val tagWidth = conf.addrBits - log2Up(sets) + val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) @@ -499,8 +499,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) - extends DRAMSideLLCLike { +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 22fa933c..eba971fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,57 +2,58 @@ package uncore import Chisel._ import scala.math._ -case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { - val ppnBits = paddrBits - pgIdxBits - val vpnBits = vaddrBits - pgIdxBits +case object PAddrBits extends Field[Int] +case object VAddrBits extends Field[Int] +case object PgIdxBits extends Field[Int] +case object ASIdBits extends Field[Int] +case object PermBits extends Field[Int] +case object PPNBits extends Field[Int] +case object VPNBits extends Field[Int] + +case object MIFAddrBits extends Field[Int] +case object MIFDataBits extends Field[Int] +case object MIFTagBits extends Field[Int] +case object MIFDataBeats extends Field[Int] + +trait HasMemData extends Bundle { + val data = Bits(width = params(MIFDataBits)) } -case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) - -abstract trait MemoryIFSubBundle extends Bundle { - implicit val conf: MemoryIFConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +trait HasMemAddr extends Bundle { + val addr = UInt(width = params(MIFAddrBits)) } -trait HasMemData extends MemoryIFSubBundle { - val data = Bits(width = conf.dataBits) +trait HasMemTag extends Bundle { + val tag = UInt(width = params(MIFTagBits)) } -trait HasMemAddr extends MemoryIFSubBundle { - val addr = UInt(width = conf.addrBits) -} - -trait HasMemTag extends MemoryIFSubBundle { - val tag = UInt(width = conf.tagBits) -} - -class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag +class MemResp extends HasMemData with HasMemTag -class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData +class MemData extends HasMemData -class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemPipeIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new MemIO().flip @@ -61,6 +62,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -68,7 +70,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -104,12 +106,12 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -127,21 +129,22 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemDesserIO(w: Int) extends Bundle { val narrow = new MemSerializedIO(w).flip val wide = new MemIO } -class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -169,13 +172,13 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -189,7 +192,7 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, conf.dataBeats)) + val dataq = Module(new Queue(new MemResp, dbeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -198,18 +201,21 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { +class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { val uncached = new UncachedTileLinkIO().flip val mem = new MemIO } - - require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) - //require(tlconf.clientXactIdBits <= mifconf.tagBits) + val co = params(TLCoherence) + val tbits = params(MIFTagBits) + val dbits = params(MIFDataBits) + val dbeats = params(MIFDataBits) + require(params(TLDataBits) == dbits*dbeats) + //require(params(TLClientXactIdBits) <= params(MIFTagBits)) val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = mifconf.dataBeats + val cnt_max = dbeats val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) val active_out = Reg(init=Bool(false)) @@ -222,7 +228,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) val active_in = Reg(init=Bool(false)) val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = mifconf.tagBits)) + val tag_in = Reg(UInt(width = tbits)) // Decompose outgoing TL Acquires into MemIO cmd and data when(!active_out && io.uncached.acquire.valid) { @@ -232,7 +238,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) + has_data := co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { @@ -240,7 +246,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon } when(mem_data_q.io.enq.fire()) { cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(mifconf.dataBits) + buf_out := buf_out >> UInt(dbits) } when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { active_out := Bool(false) @@ -264,7 +270,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) tag_in := io.mem.resp.bits.tag } when(active_in) { @@ -272,7 +278,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon active_in := Bool(false) } when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) cnt_in := cnt_in + UInt(1) } } diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 6889bd7f..72002d45 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,32 +1,35 @@ package uncore import Chisel._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) +case object LNMasters extends Field[Int] +case object LNClients extends Field[Int] +case object LNEndpoints extends Field[Int] -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) } -class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = new PhysicalHeader +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) val payload = dType.clone - override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } + override def clone = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module - -class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))}.flip + val out = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))} } -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new BasicCrossbarIO(dType) - val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) +abstract class PhysicalNetwork extends Module + +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { + val io = new BasicCrossbarIO(n, dType) + + val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits @@ -34,30 +37,26 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical }} out <> rrarb.io.out }} - for(i <- 0 until conf.nEndpoints) { + for(i <- 0 until n) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } -case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { - val nEndpoints = nMasters + nClients +abstract class LogicalNetwork extends Module + +class LogicalHeader extends Bundle { + val src = UInt(width = log2Up(params(LNEndpoints))) + val dst = UInt(width = log2Up(params(LNEndpoints))) } -abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) -} - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { val header = new LogicalHeader val payload = dType.clone override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +68,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 68da8213..d49e264b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,27 +1,29 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) +case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLAddrBits extends Field[Int] +case object TLMasterXactIdBits extends Field[Int] +case object TLClientXactIdBits extends Field[Int] +case object TLDataBits extends Field[Int] +case object TLWriteMaskBits extends Field[Int] +case object TLWordAddrBits extends Field[Int] +case object TLAtomicOpBits extends Field[Int] -abstract trait TileLinkSubBundle extends Bundle { - implicit val tlconf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] +trait HasPhysicalAddress extends Bundle { + val addr = UInt(width = params(TLAddrBits)) } -trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = tlconf.addrBits) +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = params(TLClientXactIdBits)) } -trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = tlconf.clientXactIdBits) +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = params(TLMasterXactIdBits)) } -trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = tlconf.masterXactIdBits) -} - -trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = tlconf.dataBits) +trait HasTileLinkData extends Bundle { + val data = Bits(width = params(TLDataBits)) } trait SourcedMessage extends Bundle @@ -30,7 +32,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -41,42 +43,42 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.data := data acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } - def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a acq } } -class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = tlconf.co.acquireTypeWidth) - val write_mask = Bits(width = tlconf.writeMaskBits) - val subword_addr = Bits(width = tlconf.wordAddrBits) - val atomic_opcode = Bits(width = tlconf.atomicOpBits) + val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) + val write_mask = Bits(width = params(TLWriteMaskBits)) + val subword_addr = Bits(width = params(TLWordAddrBits)) + val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } object Probe { - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr @@ -85,22 +87,22 @@ object Probe } } -class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = tlconf.co.probeTypeWidth) + val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.data := data rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -109,24 +111,24 @@ object Release rel.data := UInt(0) rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } } -class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = tlconf.co.releaseTypeWidth) + val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } object Grant { - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { val gnt = new Grant gnt.g_type := g_type gnt.client_xact_id := client_xact_id @@ -134,40 +136,35 @@ object Grant gnt.data := UInt(0) gnt } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { val gnt = apply(g_type, client_xact_id, master_xact_id) gnt.data := data gnt } } -class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = tlconf.co.grantTypeWidth) + val g_type = UInt(width = params(TLCoherence).grantTypeWidth) } -class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { - implicit val ln = conf.ln +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) - override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) - override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - +abstract class TileLinkArbiterLike(val arbN: Int) extends Module { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -178,7 +175,7 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -205,7 +202,8 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class UncachedTileLinkIOArbiter(n: Int) + extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO @@ -219,7 +217,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } -abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO @@ -267,9 +265,9 @@ abstract trait UsesNewId { in.client_xact_id } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 09bb0714..013f84c2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,33 +1,27 @@ package uncore import Chisel._ -trait CoherenceAgentConfiguration { - def tl: TileLinkConfiguration - def nReleaseTransactions: Int - def nAcquireTransactions: Int -} +case object NReleaseTransactors extends Field[Int] +case object NAcquireTransactors extends Field[Int] +case object NTransactors extends Field[Int] +case object NClients extends Field[Int] -abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { +abstract class CoherenceAgent extends Module { + val co = params(TLCoherence) val io = new Bundle { - val inner = (new TileLinkIO()(conf.tl)).flip - val outer = new UncachedTileLinkIO()(conf.tl) - val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(params(NClients)){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration( - val tl: TileLinkConfiguration, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int) extends CoherenceAgentConfiguration - -class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) + val trackerList = (0 until params(NReleaseTransactors)).map(id => + Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (params(NReleaseTransactors) until params(NTransactors)).map(id => + Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -85,12 +79,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) } -abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -100,15 +94,13 @@ abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -153,17 +145,16 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -174,12 +165,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -219,9 +210,9 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -244,7 +235,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -252,7 +243,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) From 9ab3a4262cedb689d616c6427f20f88d2e40521d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 11 Aug 2014 18:35:49 -0700 Subject: [PATCH 233/374] Cache utility traits. Completely compiles, asm tests hang. --- uncore/src/main/scala/cache.scala | 113 +++++++++++++++----------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 17 ++-- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 247110a5..21cf582e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -3,17 +3,35 @@ import Chisel._ case object NSets extends Field[Int] case object NWays extends Field[Int] -case object IsDM extends Field[Boolean] -case object TagBits extends Field[Int] -case object IdxBits extends Field[Int] -case object OffBits extends Field[Int] -case object UntagBits extends Field[Int] -case object WayBits extends Field[Int] +case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object WordBits extends Field[Int] -case object RefillCycles extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +abstract trait CacheParameters extends UsesParameters { + val paddrBits = params(PAddrBits) + val vaddrBits = params(VAddrBits) + val pgIdxBits = params(PgIdxBits) + val nSets = params(NSets) + val nWays = params(NWays) + val blockOffBits = params(BlockOffBits) + val wordBits = params(WordBits) + val rowBits = params(RowBits) + val wordOffBits = log2Up(wordBits) + val idxBits = log2Up(nSets) + val untagBits = blockOffBits + idxBits + val tagBits = paddrBits - untagBits + val wayBits = log2Up(nWays) + val isDM = nWays == 1 + val rowWords = rowBits/wordBits + val rowBytes = rowBits*8 + val rowOffBits = log2Up(rowBytes) + val refillCycles = params(TLDataBits)/rowBits +} + +abstract class CacheBundle extends Bundle with CacheParameters +abstract class CacheModule extends Module with CacheParameters + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -30,43 +48,43 @@ class RandomReplacement(ways: Int) extends ReplacementPolicy { def hit = {} } -abstract class Metadata extends Bundle { - val tag = Bits(width = params(TagBits)) +abstract class Metadata extends CacheBundle { + val tag = Bits(width = tagBits) val coh: CoherenceMetadata } -class MetaReadReq extends Bundle { - val idx = Bits(width = params(IdxBits)) +class MetaReadReq extends CacheBundle { + val idx = Bits(width = idxBits) } class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { - val way_en = Bits(width = params(WayBits)) + val way_en = Bits(width = nWays) val data = gen.clone override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} + val resp = Vec.fill(nWays){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) - val rst = rst_cnt < UInt(params(NSets)) + val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) + val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) + val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until params(NWays)) { + for (w <- 0 until nWays) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -75,14 +93,20 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { io.write.ready := !rst } -trait HasL2Id extends Bundle { - val id = UInt(width = log2Up(params(NTransactors))) +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) } -trait HasL2InternalRequestState extends Bundle { +trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() val old_meta = new L2Metadata - val way_en = Bits(width = params(NWays)) + val way_en = Bits(width = nWays) } object L2Metadata { @@ -93,23 +117,22 @@ object L2Metadata { meta } } -class L2Metadata extends Metadata { - val coh = params(TLCoherence).masterMetadataOnFlush.clone +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = params(TagBits)) + val tag = Bits(width = tagBits) } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp extends Bundle +class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray extends Module { - val (co, ways) = (params(TLCoherence), params(NWays)) +class L2MetadataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip @@ -123,7 +146,7 @@ class L2MetadataArray extends Module { val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -146,8 +169,8 @@ class L2MetadataArray extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends Bundle with HasL2Id { - val way_en = Bits(width = params(NWays)) +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) val addr = Bits(width = params(TLAddrBits)) } @@ -160,7 +183,7 @@ class L2DataResp extends Bundle with HasL2Id { val data = Bits(width = params(TLDataBits)) } -class L2DataArray extends Module { +class L2DataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -169,9 +192,9 @@ class L2DataArray extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) - val resp = (0 until params(NWays)).map { w => - val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -185,11 +208,11 @@ class L2DataArray extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent { +class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { - require(isPow2(params(NSets))) - require(isPow2(params(NWays))) - require(params(RefillCycles) == 1) + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -207,8 +230,7 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent { } -class TSHRFile(bankId: Int) extends Module { - val (co, nClients) = (params(TLCoherence), params(NClients)) +class TSHRFile(bankId: Int) extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO @@ -234,9 +256,9 @@ class TSHRFile(bankId: Int) extends Module { } // Create TSHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map { id => + val trackerList = (0 until nReleaseTransactors).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => + } ++ (nReleaseTransactors until nTransactors).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -301,8 +323,7 @@ class TSHRFile(bankId: Int) extends Module { } -abstract class L2XactTracker extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) +abstract class L2XactTracker extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eba971fd..02d74385 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,7 +209,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val co = params(TLCoherence) val tbits = params(MIFTagBits) val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBits) + val dbeats = params(MIFDataBeats) require(params(TLDataBits) == dbits*dbeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 013f84c2..580f0477 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -3,24 +3,31 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NTransactors extends Field[Int] case object NClients extends Field[Int] -abstract class CoherenceAgent extends Module { +abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) + val nReleaseTransactors = params(NReleaseTransactors) + val nAcquireTransactors = params(NAcquireTransactors) + val nTransactors = nReleaseTransactors + nAcquireTransactors + val nClients = params(NClients) +} + +abstract class CoherenceAgent extends Module + with CoherenceAgentParameters { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(params(NClients)){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput } } class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map(id => + val trackerList = (0 until nReleaseTransactors).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (params(NReleaseTransactors) until params(NTransactors)).map(id => + (nReleaseTransactors until nTransactors).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags From e26f8a6f6ae8c1da134939a9c4177752f34a5a3c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 12 Aug 2014 14:55:44 -0700 Subject: [PATCH 234/374] Fix errors in derived cache params --- uncore/src/main/scala/cache.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 21cf582e..d52e692a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -13,18 +13,19 @@ abstract trait CacheParameters extends UsesParameters { val vaddrBits = params(VAddrBits) val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) - val nWays = params(NWays) val blockOffBits = params(BlockOffBits) - val wordBits = params(WordBits) - val rowBits = params(RowBits) - val wordOffBits = log2Up(wordBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = paddrBits - untagBits + val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 + val wordBits = params(WordBits) + val wordBytes = wordBits/8 + val wordOffBits = log2Up(wordBytes) + val rowBits = params(RowBits) val rowWords = rowBits/wordBits - val rowBytes = rowBits*8 + val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) val refillCycles = params(TLDataBits)/rowBits } From e384b33cc36c8ecbb0fe5bf5f87ee7f4e84b29af Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 19 Aug 2014 15:50:20 -0700 Subject: [PATCH 235/374] don't generate a write mask for BigMem if it isn't used not needed for llc data --- uncore/src/main/scala/llc.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 3a21cdca..09ad87bb 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -44,7 +44,12 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt From dc5643b12f2e657aa95712af92dceb4effdc3eff Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 23 Aug 2014 01:19:36 -0700 Subject: [PATCH 236/374] Final parameter refactor. --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d52e692a..c35aeedf 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,6 +1,7 @@ package uncore import Chisel._ +case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] From 17b2359c9ae70882cb4e309afe7bdc47bc11f33f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 24 Aug 2014 19:27:58 -0700 Subject: [PATCH 237/374] htif parameters trait --- uncore/src/main/scala/htif.scala | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 153bad3a..bfe4ef10 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -9,7 +9,18 @@ case object HTIFNSCR extends Field[Int] case object HTIFOffsetBits extends Field[Int] case object HTIFNCores extends Field[Int] -class HostIO(val w: Int) extends Bundle +abstract trait HTIFParameters extends UsesParameters { + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) +} + +abstract class HTIFBundle extends Bundle with HTIFParameters + +class HostIO extends HTIFBundle { val clk = Bool(OUTPUT) val clk_edge = Bool(OUTPUT) @@ -25,42 +36,34 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO extends Bundle -{ +class HTIFIO extends HTIFBundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(params(HTIFNCores))) + val id = UInt(INPUT, log2Up(nCores)) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) + val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO extends Bundle -{ - val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} +class SCRIO extends HTIFBundle { + val rdata = Vec.fill(nSCR){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) + val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, 64) } -class HTIF(pcr_RESET: Int) extends Module -{ - val dataBits = params(TLDataBits) - val co = params(TLCoherence) - val w = params(HTIFWidth) - val nSCR = params(HTIFNSCR) - val offsetBits = params(HTIFOffsetBits) - val nCores = params(HTIFNCores) - val io = new Bundle { - val host = new HostIO(w) - val cpu = Vec.fill(nCores){new HTIFIO().flip} +class HTIFModuleIO extends HTIFBundle { + val host = new HostIO + val cpu = Vec.fill(nCores){new HTIFIO}.flip val mem = new TileLinkIO val scr = new SCRIO - } +} +class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { + val io = new HTIFModuleIO io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' From 712f3a754d73c3fa8149e04596d9b93dfd891658 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 2 Sep 2014 12:34:42 -0700 Subject: [PATCH 238/374] merge in master --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index bfe4ef10..8d1b2618 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -104,7 +104,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 5bc89e38..f34ffac4 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,12 +39,17 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -61,7 +66,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +389,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +399,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) @@ -438,10 +442,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From bfb662968d2e34ad589988866a4638900c5ad39e Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 14:33:58 -0700 Subject: [PATCH 239/374] fixes sbt error during first run --- uncore/chisel-dependent.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index fa8d5490..eeb52c04 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,6 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -val chiselVersion = System.getProperty("chiselVersion", "None") +lazy val chiselVersion = System.getProperty("chiselVersion", "None") libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From f8821b4cc9a9878c76b6ab849f31914163da4e04 Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 15:16:03 -0700 Subject: [PATCH 240/374] better fix with explanation of sbt issue --- uncore/chisel-dependent.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index eeb52c04..88eb615c 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,8 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -lazy val chiselVersion = System.getProperty("chiselVersion", "None") +val chiselVersion_u = System.getProperty("chiselVersion", "None") -libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) +// _u a temporary fix until sbt 13.6 https://github.com/sbt/sbt/issues/1465 + +libraryDependencies ++= ( if (chiselVersion_u != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion_u) :: Nil; else Nil) From f8d450b4e2156d73cbdb383c4eed19cbbd87ab0c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 11 Sep 2014 22:06:03 -0700 Subject: [PATCH 241/374] mark DRAMSideLLC as HasKnownBug --- uncore/src/main/scala/llc.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f34ffac4..1878a356 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -354,7 +354,19 @@ abstract class DRAMSideLLCLike extends Module { } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike +// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new +// L2$. Stay stuned. +// +// Bug description: +// There's a race condition between the writeback unit (the module which +// sends the data out to the backside interface) and the data unit (the module +// which writes the data into the SRAM in the L2$). The "hit status" +// is saved in a register, however, is updated again when there's +// a transaction coming from the core without waiting until the writeback unit +// has sent out all its data to the outer memory system. This is why the +// problem manifests at a higher probability with the slow backup memory port. + +class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -505,7 +517,6 @@ object HellaQueue } class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 From 0b51d70bd288d9146c5c8d99d5916a208c82789b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:31:38 -0700 Subject: [PATCH 242/374] add LICENSE --- uncore/src/main/scala/cache.scala | 2 ++ uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/consts.scala | 2 ++ uncore/src/main/scala/htif.scala | 2 ++ uncore/src/main/scala/llc.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 2 ++ uncore/src/main/scala/network.scala | 2 ++ uncore/src/main/scala/package.scala | 2 ++ uncore/src/main/scala/slowio.scala | 2 ++ uncore/src/main/scala/tilelink.scala | 2 ++ uncore/src/main/scala/uncore.scala | 2 ++ 11 files changed, 22 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c35aeedf..686f0ac1 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 45a1970c..5fa0b7a6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 64d6c3a7..a402e7db 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore package constants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8d1b2618..edcd66cb 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1878a356..35553702 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 02d74385..6f42f010 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ import scala.math._ diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 72002d45..d3f4a9e8 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 501d511d..2c6c4a5f 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) diff --git a/uncore/src/main/scala/slowio.scala b/uncore/src/main/scala/slowio.scala index 9ed0abb7..95ca34e6 100644 --- a/uncore/src/main/scala/slowio.scala +++ b/uncore/src/main/scala/slowio.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d49e264b..af736c97 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 580f0477..8f1a7a67 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ From 49b027db2c8a423b76cb63bf19089f5077823138 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:36:29 -0700 Subject: [PATCH 243/374] forgot to add LICENSE file --- uncore/LICENSE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 uncore/LICENSE diff --git a/uncore/LICENSE b/uncore/LICENSE new file mode 100644 index 00000000..7cff15e4 --- /dev/null +++ b/uncore/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. From f249da1803d958804e5d1e715e324c11b4c1a913 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 17 Sep 2014 11:25:14 -0700 Subject: [PATCH 244/374] update README --- uncore/README | 6 ------ uncore/README.md | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 uncore/README create mode 100644 uncore/README.md diff --git a/uncore/README b/uncore/README deleted file mode 100644 index c920e19a..00000000 --- a/uncore/README +++ /dev/null @@ -1,6 +0,0 @@ -This is the repository for uncore components assosciated with UCB_BAR chip -projects. To uses these modules, include this repo as a git submodule within -the your chip repository and add it as Project in your chip's build.scala. -These components are only dependent on Chisel, i.e. -lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) - diff --git a/uncore/README.md b/uncore/README.md new file mode 100644 index 00000000..003f9caa --- /dev/null +++ b/uncore/README.md @@ -0,0 +1,11 @@ +Uncore Library +============== + +This is the repository for uncore components assosciated with Rocket chip +project. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. + + lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + +Documentation about the uncore library will come in the near future. From f7b1e23eadebdc956b68dbd6f5e8eedb04bb6241 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 16:59:48 -0700 Subject: [PATCH 245/374] functional style on MuxBundle --- uncore/src/main/scala/coherence.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5fa0b7a6..1a3c4573 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -5,11 +5,7 @@ import Chisel._ object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - var res = default; - for ((t, v) <- mapping.reverse){ - res = Mux(t, v, res); - } - res + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) } } From faed47d131211fa12c393ae5fd864319cfc90ffc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:01:19 -0700 Subject: [PATCH 246/374] use thunk for dir info --- uncore/src/main/scala/coherence.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1a3c4573..abacdabd 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir.clone + val sharers = c.dir() override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -75,10 +75,12 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation { def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } def none(dummy: Int = 0) = sharers === UInt(0) def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + def count(dummy: Int = 0) = PopCount(sharers) + def next(dummy: Int = 0) = PriorityEncoder(sharers) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -144,10 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) +abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) with UncachedTransactions -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -299,7 +301,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncac def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -470,7 +472,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -643,7 +645,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -822,7 +824,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnc def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 def nMasterStates = 3 def nAcquireTypes = 8 From 149d51d644d1ffb0ae7914d512f0c7b41951ff71 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:10:32 -0700 Subject: [PATCH 247/374] more coherence API cleanup --- uncore/src/main/scala/cache.scala | 11 +-- uncore/src/main/scala/coherence.scala | 102 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 8 +- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 686f0ac1..faa879a9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,7 +376,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) @@ -454,11 +454,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index abacdabd..bd6fecc3 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,7 +28,7 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers := m.sharers.flush() + m.sharers.flush() m } def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { @@ -51,32 +51,36 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val sharers: UInt + val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation def none(dummy: Int = 0): Bool def one(dummy: Int = 0): Bool + def count(dummy: Int = 0): UInt + def next(dummy: Int = 0): UInt } class NullRepresentation extends DirectoryRepresentation { - val sharers = UInt(0) + val internal = UInt(0) def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) + def count(dummy: Int = 0) = UInt(0) + def next(dummy: Int = 0) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val sharers = UInt(width = nClients) - def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } - def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } - def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = sharers === UInt(0) - def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) - def count(dummy: Int = 0) = PopCount(sharers) - def next(dummy: Int = 0) = PriorityEncoder(sharers) + val internal = UInt(width = nClients) + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def push(id: UInt) = { internal := internal | UIntToOH(id); this } + def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = internal === UInt(0) + def one(dummy: Int = 0) = PopCount(internal) === UInt(1) + def count(dummy: Int = 0) = PopCount(internal) + def next(dummy: Int = 0) = PriorityEncoder(internal) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } @@ -113,12 +117,14 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt - def getGrantType(a_type: UInt, count: UInt): UInt - def getGrantType(rel: Release, count: UInt): UInt + def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt + def getGrantType(a: Acquire, m: MasterMetadata): UInt + def getGrantType(r: Release, m: MasterMetadata): UInt + //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO + //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -259,8 +265,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -270,14 +276,14 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, acquireWriteUncached -> probeInvalidate, @@ -427,8 +433,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -438,15 +444,15 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -604,9 +610,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -615,14 +621,14 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -778,9 +784,9 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -789,15 +795,15 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -975,9 +981,9 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -987,15 +993,15 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8f1a7a67..2e62d79f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -130,7 +130,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -161,7 +161,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -195,11 +194,11 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -219,7 +218,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) From 53b8d7b03164a97c5bb03cd6910b978168cf1252 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 13 Jul 2014 22:29:15 -0700 Subject: [PATCH 248/374] use new coherence methods in l2, ready to query dir logic --- uncore/src/main/scala/cache.scala | 83 ++++++++++++++++++++------- uncore/src/main/scala/coherence.scala | 2 +- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index faa879a9..3de8ba4a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -109,7 +109,7 @@ trait HasL2Id extends Bundle with CoherenceAgentParameters { trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() - val old_meta = new L2Metadata + val meta = new L2Metadata val way_en = Bits(width = nWays) } @@ -167,7 +167,7 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match - io.resp.bits.old_meta := Mux(s2_tag_match, + io.resp.bits.meta := Mux(s2_tag_match, L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) @@ -354,18 +354,16 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val new_meta = Reg(new L2Metadata) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), @@ -376,32 +374,47 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src + xact := c_rel.payload + init_client_id := c_rel.header.src state := s_mem } - } -/* + }/* is(s_meta_read) { when(io.meta_read.ready) state := s_meta_resp } is(s_meta_resp) { when(io.meta_resp.valid) { - new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits - old_meta := io.meta.resp.bits.old_meta + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) state := Mux(s_meta_write -Mux(co.messageHasData(xact), s_mem, s_ack) - } - */ + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -417,11 +430,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) - val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -447,7 +460,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready @@ -459,7 +471,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -471,6 +483,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -479,7 +513,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) @@ -490,6 +523,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(needs_outer_read, s_mem_read, s_make_grant)) } } + // s_read_meta + // xact_internal := resp is(s_probe) { // Generate probes io.inner.probe.valid := probe_flags.orR @@ -499,6 +534,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data to be written back when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index bd6fecc3..7dbff4aa 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -74,7 +74,7 @@ class NullRepresentation extends DirectoryRepresentation { class FullRepresentation(nClients: Int) extends DirectoryRepresentation { val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? def push(id: UInt) = { internal := internal | UIntToOH(id); this } def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } def none(dummy: Int = 0) = internal === UInt(0) From 82fe22f958a812cf8ff1d7755d5d4cb01cb7c242 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 11:14:36 -0700 Subject: [PATCH 249/374] support for multiple tilelink paramerterizations in same design Conflicts: src/main/scala/cache.scala --- uncore/src/main/scala/cache.scala | 46 ++++++++++++++++------------ uncore/src/main/scala/tilelink.scala | 5 +-- uncore/src/main/scala/uncore.scala | 44 ++++++++++++++------------ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3de8ba4a..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -212,13 +212,14 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) require(refillCycles == 1) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) @@ -234,10 +235,10 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParamete } -class TSHRFile(bankId: Int) extends L2HellaCacheModule { +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) @@ -261,9 +262,9 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } // Propagate incoherence flags @@ -312,7 +313,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { ack.ready := Bool(true) // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out @@ -327,10 +329,10 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { } -abstract class L2XactTracker extends L2HellaCacheModule { +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -350,7 +352,7 @@ abstract class L2XactTracker extends L2HellaCacheModule { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -364,10 +366,11 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -426,7 +429,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -440,11 +443,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af736c97..8bb98f0a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ +case object TLId extends Field[String] case object TLCoherence extends Field[CoherencePolicyWithUncached] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] @@ -155,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO extends UncachedTileLinkIO { +class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 2e62d79f..37e535f8 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,22 +15,23 @@ abstract trait CoherenceAgentParameters extends UsesParameters { val nClients = params(NClients) } -abstract class CoherenceAgent extends Module +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module with CoherenceAgentParameters { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput } } -class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { +class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) { // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId))) + Module(new AcquireTracker(id, bankId, innerId, outerId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -82,17 +83,18 @@ class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } -abstract class XactTracker extends Module { +abstract class XactTracker(innerId: String, outerId: String) extends Module { val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -105,7 +107,7 @@ abstract class XactTracker extends Module { val m_gnt = io.outer.grant.bits } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -120,10 +122,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -154,7 +157,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { } } -class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { +class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -167,11 +170,14 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) From 7571695320f3804387a87aec570cf7ba80c01da6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 15:11:24 -0700 Subject: [PATCH 250/374] Removed broken or unfinished modules, new MemPipeIO converter --- uncore/src/main/scala/bigmem.scala | 80 ++++ uncore/src/main/scala/cache.scala | 499 ----------------------- uncore/src/main/scala/llc.scala | 554 -------------------------- uncore/src/main/scala/memserdes.scala | 109 +++++ 4 files changed, 189 insertions(+), 1053 deletions(-) create mode 100644 uncore/src/main/scala/bigmem.scala delete mode 100644 uncore/src/main/scala/llc.scala diff --git a/uncore/src/main/scala/bigmem.scala b/uncore/src/main/scala/bigmem.scala new file mode 100644 index 00000000..fbbc4c07 --- /dev/null +++ b/uncore/src/main/scala/bigmem.scala @@ -0,0 +1,80 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module +{ + class Inputs extends Bundle { + val addr = UInt(INPUT, log2Up(n)) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = Valid(new Inputs).flip + val rdata = gen.asOutput + } + val data = gen + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} + for (i <- 0 until nDeep) { + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(next=ren) + val rdata = Vec.fill(nWide){Bits()} + + val r = Pipe(ren, in.bits.addr, postLatency) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val ridx = if (postLatency > 0) Reg(Bits()) else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + when (in.valid) { + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } + .otherwise { if (postLatency > 0) ridx := idx } + } + + if (postLatency == 0) { + dout = mem(idx) + } else if (postLatency == 1) { + dout = mem(ridx) + } else + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits + + rdata(j) := dout + } + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := r.valid + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..26157a75 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,502 +96,3 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } - -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters - -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters - -trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) -} - -trait HasL2InternalRequestState extends L2HellaCacheBundle { - val tag_match = Bool() - val meta = new L2Metadata - val way_en = Bits(width = nWays) -} - -object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { - val meta = new L2Metadata - meta.tag := tag - meta.coh := coh - meta - } -} -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone -} - -class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = tagBits) -} - -class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - -class L2MetaResp extends L2HellaCacheBundle - with HasL2Id - with HasL2InternalRequestState - -class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } - - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) - meta.io.read <> io.read - meta.io.write <> io.write - - val s1_clk_en = Reg(next = io.read.fire()) - val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) - val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) - val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits - val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) - val s2_tag_match = s2_tag_match_way.orR - val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - - val replacer = params(Replacer)() - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - - io.resp.valid := Reg(next = s1_clk_en) - io.resp.bits.id := RegEnable(s1_id, s1_clk_en) - io.resp.bits.tag_match := s2_tag_match - io.resp.bits.meta := Mux(s2_tag_match, - L2Metadata(s2_repl_meta.tag, s2_hit_coh), - s2_repl_meta) - io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) -} - -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { - val way_en = Bits(width = nWays) - val addr = Bits(width = params(TLAddrBits)) -} - -class L2DataWriteReq extends L2DataReadReq { - val wmask = Bits(width = params(TLWriteMaskBits)) - val data = Bits(width = params(TLDataBits)) -} - -class L2DataResp extends Bundle with HasL2Id { - val data = Bits(width = params(TLDataBits)) -} - -class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } - - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) - } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) - - io.read.ready := Bool(true) - io.write.ready := Bool(true) -} - -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - - require(isPow2(nSets)) - require(isPow2(nWays)) - require(refillCycles == 1) - - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) - - tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp - io.outer <> tshrfile.io.outer - io.incoherent <> tshrfile.io.incoherent -} - - -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } - } - - // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) - } - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Handle acquire transaction initiation - val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires - - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - - // Handle releases, which might be voluntary and might have data - val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - io.outer <> outer_arb.io.out - - // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) - -} - - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - - io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) - xact.client_xact_id, - UInt(trackerId)) - - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - state := s_mem - } - }/* - is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp - } - is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } - } - is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } - } - } -} - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) - val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) - - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - m_gnt.payload.data) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) - when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) - } - } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) - when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } - } - } -} diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala deleted file mode 100644 index 35553702..00000000 --- a/uncore/src/main/scala/llc.scala +++ /dev/null @@ -1,554 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module -{ - class Inputs extends Bundle { - val addr = UInt(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = Valid(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 - val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec.fill(nDeep){Bits()} - val rdataSel = Vec.fill(nDeep){Bool()} - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(next=ren) - val rdata = Vec.fill(nWide){Bits()} - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val ridx = if (postLatency > 0) Reg(Bits()) else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - when (in.valid) { - when (in.bits.rw) { - if (noMask) - mem.write(idx, wdata0) - else - mem.write(idx, wdata0, wmask0) - } - .otherwise { if (postLatency > 0) ridx := idx } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = mem(ridx) - } else - dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UInt(width = log2Up(ways)) - val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCTagReq(ways: Int) extends HasMemAddr -{ - val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Decoupled(new MemReqCmd).flip - val repl_way = UInt(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) - val data = Decoupled(new LLCDataReq(ways)) - val tag = Decoupled(new LLCTagReq(ways)) - val mem = new MemPipeIO - val mem_resp_set = UInt(OUTPUT, log2Up(sets)) - val mem_resp_way = UInt(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) - val way = UInt(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UInt(width = log2Up(refill_cycles)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) - val wb_busy = Bool() - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec.fill(outstanding){Reg(init=Bool(false))} - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec.fill(outstanding){Reg(new MSHR)} - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).wb_busy := Bool(false) - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UInt(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR && io.data.ready // allow in-flight hits to drain - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { - mshr(writebackId).old_dirty := Bool(false) - mshr(writebackId).wb_busy := Bool(true) - } - mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } - val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new MemPipeIO - } - - val valid = Reg(init=Bool(false)) - val who = Reg(UInt()) - val addr = Reg(UInt()) - val cmd_sent = Reg(Bool()) - val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UInt(1) - when (count === UInt(refill_cycles-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module -{ - val io = new Bundle { - val req = Decoupled(new LLCDataReq(ways)).flip - val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = params(MIFAddrBits))) - val writeback_data = Decoupled(new MemData) - val resp = Decoupled(new MemResp) - val mem_resp = Valid(new MemResp).flip - val mem_resp_set = UInt(INPUT, log2Up(sets)) - val mem_resp_way = UInt(INPUT, log2Up(ways)) - } - - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = Module(new Queue(new QEntry, latency+2)) - val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = Reg(init=Bool(false)) - val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UInt(1) - when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Vec.fill(n){new MemIO().flip} - val mem = new MemIO - } - - val lock = Reg(init=Bool(false)) - val locker = Reg(UInt()) - - val arb = Module(new RRArbiter(new MemReqCmd, n)) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) - for (i <- 0 until n) { - val me = UInt(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UInt(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 - when (unlock) { lock := Bool(false) } -} - -abstract class DRAMSideLLCLike extends Module { - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } -} - -// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new -// L2$. Stay stuned. -// -// Bug description: -// There's a race condition between the writeback unit (the module which -// sends the data out to the backside interface) and the data unit (the module -// which writes the data into the SRAM in the L2$). The "hit status" -// is saved in a register, however, is updated again when there's -// a transaction coming from the core without waiting until the writeback unit -// has sent out all its data to the outer memory system. This is why the -// problem manifests at a higher probability with the slow backup memory port. - -class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike -{ - val tagWidth = params(MIFAddrBits) - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) - val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) - val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) - val writeback = Module(new LLCWriteback(2, refill_cycles)) - - val initCount = Reg(init=UInt(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UInt(1) } - - val replay_s2 = Reg(init=Bool(false)) - val s2_valid = Reg(init=Bool(false)) - val s2 = Reg(new MemReqCmd) - val s3_rdy = Bool() - val replay_s2_rdy = Bool() - - val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) - val s1 = Reg(new MemReqCmd) - when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } - when (replay_s2 && replay_s2_rdy) { s1 := s2 } - - s2_valid := s1_valid - replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} - when (s1_valid) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUInt(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUInt - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - - val tag_we = initialize || setDirty || mshr.io.tag.fire() - val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) - val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) - tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we - tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) - tags.io.in.bits.rw := tag_we - tags.io.in.bits.wdata := Fill(ways, tag_wdata) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !s1_valid && !s2_valid - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) - replay_s2_rdy := s3_rdy && !tag_we - - io.cpu.resp <> data.io.resp - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data - io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || - io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) -} - -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) - - val do_flow = Bool() - val do_enq = io.enq.fire() && !do_flow - val do_deq = io.deq.fire() && !do_flow - - val maybe_full = Reg(init=Bool(false)) - val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 - when (do_enq != do_deq) { maybe_full := do_enq } - - val ptr_match = enq_ptr === deq_ptr - val empty = ptr_match && !maybe_full - val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) - do_flow := empty && io.deq.ready - - val ram = Mem(data, entries, seqRead = true) - val ram_addr = Reg(Bits()) - val ram_out_valid = Reg(Bool()) - ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits } - when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { - ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) - } - - io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) - io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) -} - -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - - val fq = Module(new HellaFlowQueue(entries)(data)) - io.enq <> fq.io.enq - io.deq <> Queue(fq.io.deq, 1, pipe = true) -} - -object HellaQueue -{ - def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { - val q = Module((new HellaQueue(entries)) { enq.bits.clone }) - q.io.enq.valid := enq.valid // not using <> so that override is allowed - q.io.enq.bits := enq.bits - enq.ready := q.io.enq.ready - q.io.deq - } -} - -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles - val size = log2Down(numEntries) + 1 - - val inc = Bool() - val dec = Bool() - val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) - - when (inc && !dec) { - count := count + UInt(1) - } - when (!inc && dec) { - count := count - UInt(refillCycles) - } - when (inc && dec) { - count := count - UInt(refillCycles-1) - } - - val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark - - io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask - io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask - io.mem.req_cmd.bits := io.cpu.req_cmd.bits - - io.mem.req_data <> io.cpu.req_data - - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq - - inc := resp_dataq.io.deq.fire() - dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw -} diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 6f42f010..b470df49 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -286,3 +286,112 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { } } +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(init=Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) + do_flow := empty && io.deq.ready + + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + + val fq = Module(new HellaFlowQueue(entries)(data)) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val cpu = new MemIO().flip + val mem = new MemPipeIO + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(init=UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) + + when (inc && !dec) { + count := count + UInt(1) + } + when (!inc && dec) { + count := count - UInt(refillCycles) + } + when (inc && dec) { + count := count - UInt(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} + +class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemPipeIO + } + + val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + a.io.uncached <> io.uncached + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + a.io.mem.resp <> b.io.cpu.resp + b.io.mem <> io.mem +} From d735f641102afb7805401dbc32c7e36a0b265904 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 2 Oct 2014 16:47:35 -0700 Subject: [PATCH 251/374] Parameter API update --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 8bb98f0a..51864fe1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -156,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } From dc1a61264df319c05ac4c1d19eadabce9adfb7b6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Sep 2014 14:48:02 -0700 Subject: [PATCH 252/374] initial version, acts like old hub --- uncore/src/main/scala/cache.scala | 499 ++++++++++++++++++++++++++++++ 1 file changed, 499 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 26157a75..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,3 +96,502 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } + +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) +} + +trait HasL2InternalRequestState extends L2HellaCacheBundle { + val tag_match = Bool() + val meta = new L2Metadata + val way_en = Bits(width = nWays) +} + +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata) = { + val meta = new L2Metadata + meta.tag := tag + meta.coh := coh + meta + } +} +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone +} + +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = tagBits) +} + +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) + with HasL2Id + +class L2MetaResp extends L2HellaCacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetadataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) + + val replacer = params(Replacer)() + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.meta := Mux(s2_tag_match, + L2Metadata(s2_repl_meta.tag, s2_hit_coh), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) + val addr = Bits(width = params(TLAddrBits)) +} + +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) +} + +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) +} + +class L2DataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Valid(new L2DataResp) + } + + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { + + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) + + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val meta = Module(new L2MetadataArray) + val data = Module(new L2DataArray) + + tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp + io.outer <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + } + + // Create TSHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + } ++ (nReleaseTransactors until nTransactors).map { id => + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + } + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.inner.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Handle releases, which might be voluntary and might have data + val release = io.inner.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + + // Free finished transactions + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Arbitrate for the outer memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + +} + + +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) + + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data), + { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + xact.client_xact_id, + UInt(trackerId)) + + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact := c_rel.payload + init_client_id := c_rel.header.src + state := s_mem + } + }/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) + state := Mux(s_meta_write + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ + is(s_mem) { + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + //TODO: Will need id reg for merged release xacts + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.requiresOuterRead(xact.a_type) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) + + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + if (nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + when( io.inner.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + probe_flags := probe_initial_flags + if(nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + // s_read_meta + // xact_internal := resp + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) + when(co.messageHasData(c_rel.payload)) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) + } + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From 394eb38a963588427fe900c7bffa369dc494ab88 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Oct 2014 01:06:33 -0700 Subject: [PATCH 253/374] temp; converted voluntary wb tracker --- uncore/src/main/scala/cache.scala | 56 ++++++++++++++------------- uncore/src/main/scala/coherence.scala | 4 +- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..b4e210b0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -122,7 +122,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone + val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -306,11 +306,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + // Free finished transactions on ack + val finish = io.inner.finish + val finish_idx = finish.bits.payload.master_xact_id + trackerList.zipWithIndex.map { case (t, i) => + t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) + } + trackerList.map(_.io.inner.finish.bits := finish.bits) + finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), @@ -353,7 +356,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -365,19 +368,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), xact.client_xact_id, UInt(trackerId)) @@ -397,8 +395,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -406,25 +403,32 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou when( io.inner.release.valid ) { xact := c_rel.payload init_client_id := c_rel.header.src - state := s_mem + state := s_meta_read } - }/* + } is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) + xact_internal := io.meta_resp.bits + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + s_busy, s_idle) + } + } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..778a04f6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -62,10 +62,10 @@ abstract class DirectoryRepresentation extends Bundle { } class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) + val internal = UInt() def pop(id: UInt) = this def push(id: UInt) = this - def flush(dummy: Int = 0) = this + def flush(dummy: Int = 0) = { internal := UInt(0); this } def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) def count(dummy: Int = 0) = UInt(0) From 86bdbd65358e253f7742927f74da69c7e9af494e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 7 Oct 2014 22:33:10 -0700 Subject: [PATCH 254/374] new tshrs, compiles but does not elaborate --- uncore/src/main/scala/cache.scala | 275 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 204 ++++++++++--------- uncore/src/main/scala/tilelink.scala | 4 + 3 files changed, 293 insertions(+), 190 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b4e210b0..9cc99e4e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -353,10 +353,14 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + def mergeData(acq: Acquire, data: UInt): UInt = { + //TODO wite mask + Mux(co.messageHasData(acq), acq.data, data) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -412,18 +416,26 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := s_meta_write + xact_internal := co.masterMetadataOnRelease(xact, + io.meta_resp.bits.meta.coh, + init_client_id) + state := Mux(io.meta_resp.bits.tag_match, + Mux(co.messageHasData(xact), s_data_write, s_meta_write), + s_grant) } } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { state := s_meta_write } + } is(s_meta_write) { io.meta_write.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + when(io.meta_write.ready) { state := s_grant } } - is(s_ack) { + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), s_busy, s_idle) } } @@ -434,61 +446,71 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) + val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val pending_probes = Reg(init = co.dir()) + val curr_p_id = pending_probes.next() + + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = co.needsWriteback(xact_internal.meta.coh) + val is_hit = co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + val c_rel_had_data = Reg{Bool()} + val c_rel_was_voluntary = Reg{Bool()} + val wb_buffer = Reg{xact.data.clone} + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - + xact.addr, + UInt(trackerId), + xact.data), { case TLId => outerId }) + val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, + Cat(xact_internal.meta.tag, + xact.addr(untagBits-1,blockOffBits)), + UInt(trackerId), + wb_buffer), { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, + xact.addr, + UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready + io.outer.grant.ready := Bool(true) //grant.data -> xact.data + val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + xact.addr, + UInt(trackerId)) + val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, + xact.addr, + UInt(trackerId)) + //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Mux(needs_writeback, + inner_probe_wb, + inner_probe_cacq) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, xact_internal.meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + xact.data) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -511,91 +533,154 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := xact_internal.meta.coh switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + state := s_meta_read } } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + is(s_meta_read) { + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + val coh = io.meta_resp.bits.meta.coh + val _tag_match = io.meta_resp.bits.tag_match + val _needs_writeback = co.needsWriteback(coh) + val _is_hit = co.isHit(xact, coh) + val _needs_probes = co.requiresProbes(xact.a_type, coh) + xact_internal := io.meta_resp.bits + when(!_needs_writeback) { + xact_internal.meta.coh := co.masterMetadataOnFlush + } + when(_needs_probes) { + pending_probes := coh.sharers + release_count := coh.sharers.count() + c_rel_had_data := Bool(false) + c_rel_was_voluntary := Bool(false) + } + state := Mux(_tag_match, + Mux(_is_hit, + Mux(_needs_probes, s_probe, s_data_read_hit), + Mux(_needs_probes, s_probe, s_outer_read)), + Mux(_needs_writeback, + Mux(_needs_probes, s_probe, s_data_read_wb), + s_outer_read)) } + } + is(s_probe) { + val skip = io.tile_incoherent(curr_p_id) || + ((curr_p_id === init_client_id) && + !co.requiresSelfProbe(xact.a_type)) + io.inner.probe.valid := !(pending_probes.none() || skip) + when(io.inner.probe.ready || skip) { + pending_probes.pop(curr_p_id) + } + when(skip) { release_count := release_count - UInt(1) } - // Handle releases, which may have data to be written back + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) + xact_internal.meta.coh := co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } + c_rel_had_data := Bool(true) + when(tag_match) { + xact.data := mergeData(xact, io.inner.release.bits.payload.data) + } .otherwise { + wb_buffer := io.inner.release.bits.payload.data } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + } + when(co.isVoluntary(c_rel.payload)) { + c_rel_was_voluntary := Bool(true) + } + when(!co.isVoluntary(c_rel.payload)) { + release_count := release_count - Mux(skip, UInt(2), UInt(1)) when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + state := Mux(tag_match, + Mux(is_hit, + Mux(c_rel_had_data, s_meta_write, s_data_read_hit), + s_outer_read), + Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) } } } } - is(s_mem_read) { + is(s_data_read_wb) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_wb + } + } + is(s_data_resp_wb) { + when(io.data_resp.valid) { + wb_buffer := io.data_resp.bits + state := s_outer_write_wb + } + } + is(s_outer_write_wb) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_wb + when(io.outer.acquire.ready) { + state := s_outer_read + } + } + is(s_outer_read) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := s_outer_resp } } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) + is(s_outer_resp) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + xact.data := mergeData(xact, io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_internal.meta.coh + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) } } - is(s_make_grant) { + is(s_data_read_hit) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_hit + } + } + is(s_data_resp_hit) { + when(io.data_resp.valid) { + xact.data := mergeData(xact, io.data_resp.bits.data) + state := s_meta_write + } + } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) + when(io.meta_write.ready) { state := s_grant } + } + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), + s_busy, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 778a04f6..5dd87b35 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -106,6 +105,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool + def needsWriteback(m: MasterMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata @@ -114,33 +114,33 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata def masterMetadataOnFlush: MasterMetadata def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata + def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool + def requiresProbes(a_type: UInt, m: MasterMetadata): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } trait UncachedTransactions { @@ -191,6 +191,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -214,16 +215,19 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached )) } @@ -292,6 +296,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -299,11 +304,10 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -346,6 +350,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -374,18 +379,20 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -462,6 +469,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -469,25 +477,25 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -522,6 +530,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -549,20 +558,21 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -612,7 +622,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -635,6 +645,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireWriteUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -642,25 +653,25 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -695,6 +706,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -723,20 +735,21 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -786,7 +799,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -813,6 +826,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -820,26 +834,26 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -873,6 +887,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -914,25 +929,24 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientMigratoryDirty -> clientInvalid)) )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), - popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m, - is(releaseDowngradeDataMigratory) -> m, - is(releaseDowngradeAckHasCopy) -> m, - is(releaseInvalidateDataMigratory) -> next, - is(releaseInvalidateAckMigratory) -> next + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next, + r.is(releaseInvalidateDataMigratory) -> next, + r.is(releaseInvalidateAckMigratory) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached, + g.is(grantReadMigratory) -> cached )) } @@ -983,7 +997,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1012,6 +1026,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> probeInvalidateOthers )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) @@ -1019,12 +1034,11 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..46ae899e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -77,6 +77,7 @@ class Acquire extends ClientSourcedMessage val write_mask = Bits(width = params(TLWriteMaskBits)) val subword_addr = Bits(width = params(TLWordAddrBits)) val atomic_opcode = Bits(width = params(TLAtomicOpBits)) + def is(t: UInt) = a_type === t } object Probe @@ -94,6 +95,7 @@ class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + def is(t: UInt) = p_type === t } object Release @@ -127,6 +129,7 @@ class Release extends ClientSourcedMessage with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + def is(t: UInt) = r_type === t } object Grant @@ -151,6 +154,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + def is(t: UInt) = g_type === t } class Finish extends ClientSourcedMessage with HasMasterTransactionId From 044b19dbc1de7dfeb1d9c15a76e937ef4bdc0a0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Oct 2014 11:46:35 -0700 Subject: [PATCH 255/374] Compiles and elaborates, does not pass asm tests --- uncore/src/main/scala/cache.scala | 31 +++++---- uncore/src/main/scala/coherence.scala | 93 +++++++++++++-------------- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9cc99e4e..1c6fc044 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -130,8 +130,9 @@ class L2MetaReadReq extends MetaReadReq with HasL2Id { } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - + with HasL2Id { + override def clone = new L2MetaWriteReq().asInstanceOf[this.type] +} class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -360,7 +361,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -399,7 +400,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -450,21 +450,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } + val test = Reg{UInt()} val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts val release_count = Reg(init = UInt(0, width = log2Up(nClients))) - val pending_probes = Reg(init = co.dir()) - val curr_p_id = pending_probes.next() + val pending_probes = Reg(init = co.dir().flush) + val curr_p_id = co.dir().next(pending_probes) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - val c_rel_had_data = Reg{Bool()} - val c_rel_was_voluntary = Reg{Bool()} + val c_rel_had_data = Reg(init = Bool(false)) + val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && @@ -557,12 +558,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits + test := UInt(0) when(!_needs_writeback) { - xact_internal.meta.coh := co.masterMetadataOnFlush +// xact_internal.meta.coh := co.masterMetadataOnFlush + test := UInt(12) } when(_needs_probes) { pending_probes := coh.sharers - release_count := coh.sharers.count() + release_count := co.dir().count(coh.sharers) c_rel_had_data := Bool(false) c_rel_was_voluntary := Bool(false) } @@ -579,19 +582,21 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val skip = io.tile_incoherent(curr_p_id) || ((curr_p_id === init_client_id) && !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(pending_probes.none() || skip) + io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) when(io.inner.probe.ready || skip) { - pending_probes.pop(curr_p_id) + co.dir().pop(pending_probes, curr_p_id) } when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { +/* xact_internal.meta.coh := co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, c_rel.header.src) +*/ when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { @@ -623,7 +628,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_resp_wb) { when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits + wb_buffer := io.data_resp.bits.data state := s_outer_write_wb } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5dd87b35..2bb6ec01 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,10 +28,10 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers.flush() + m.sharers := c.dir().flush m } - def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = apply(state) m.sharers := sharers m @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir() + val sharers = UInt(width = c.dir().width) override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -50,37 +50,34 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe } */ -abstract class DirectoryRepresentation extends Bundle { - def pop(id: UInt): DirectoryRepresentation - def push(id: UInt): DirectoryRepresentation - def flush(dummy: Int = 0): DirectoryRepresentation - def none(dummy: Int = 0): Bool - def one(dummy: Int = 0): Bool - def count(dummy: Int = 0): UInt - def next(dummy: Int = 0): UInt +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt() - def pop(id: UInt) = this - def push(id: UInt) = this - def flush(dummy: Int = 0) = { internal := UInt(0); this } - def none(dummy: Int = 0) = Bool(false) - def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) - def next(dummy: Int = 0) = UInt(0) +class NullRepresentation extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(0) + def next(s: UInt) = UInt(0) } -class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? - def push(id: UInt) = { internal := internal | UIntToOH(id); this } - def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = internal === UInt(0) - def one(dummy: Int = 0) = PopCount(internal) === UInt(1) - def count(dummy: Int = 0) = PopCount(internal) - def next(dummy: Int = 0) = PriorityEncoder(internal) - override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -216,7 +213,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -224,7 +221,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached @@ -307,7 +304,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -380,7 +377,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -388,7 +385,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached, @@ -480,7 +477,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -559,7 +556,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -567,7 +564,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -622,7 +619,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -656,7 +653,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -736,7 +733,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -744,7 +741,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -799,7 +796,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -838,7 +835,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -930,7 +927,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -940,7 +937,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -997,7 +994,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1038,7 +1035,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From a891ba1d460c832d97d8e5b6e7865344eb997755 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Oct 2014 17:40:30 -0700 Subject: [PATCH 256/374] more correct handling of internal state --- uncore/src/main/scala/cache.scala | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1c6fc044..4ca6b53e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,11 +459,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) + + val next_coh_on_release = co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) + val next_coh_on_grant = co.masterMetadataOnGrant( + c_gnt.payload, + xact_internal.meta.coh, + c_gnt.header.dst) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + val c_rel_had_data = Reg(init = Bool(false)) val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} @@ -535,7 +546,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := xact_internal.meta.coh + io.meta_write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -558,11 +569,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits - test := UInt(0) - when(!_needs_writeback) { -// xact_internal.meta.coh := co.masterMetadataOnFlush - test := UInt(12) - } when(_needs_probes) { pending_probes := coh.sharers release_count := co.dir().count(coh.sharers) @@ -591,12 +597,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { -/* - xact_internal.meta.coh := co.masterMetadataOnRelease( - c_rel.payload, - xact_internal.meta.coh, - c_rel.header.src) -*/ + xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { From 27c72e5eedd708fe98c73aabd39124f48a099177 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Oct 2014 21:50:03 -0700 Subject: [PATCH 257/374] nearly all isa tests pass --- uncore/src/main/scala/cache.scala | 152 ++++++++++++++------------ uncore/src/main/scala/coherence.scala | 14 +++ uncore/src/main/scala/memserdes.scala | 2 +- 3 files changed, 97 insertions(+), 71 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4ca6b53e..6b5e9ba8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -157,13 +157,13 @@ class L2MetadataArray extends L2HellaCacheModule { val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + when(!s2_tag_match) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) @@ -197,9 +197,9 @@ class L2DataArray extends L2HellaCacheModule { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -258,7 +258,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } } // Create TSHRs for outstanding transactions @@ -295,8 +295,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -361,7 +360,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -373,11 +372,12 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), @@ -399,7 +399,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + xact_internal.meta.coh, + init_client_id) switch (state) { is(s_idle) { @@ -416,9 +419,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := co.masterMetadataOnRelease(xact, - io.meta_resp.bits.meta.coh, - init_client_id) + xact_internal := io.meta_resp.bits state := Mux(io.meta_resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) @@ -440,6 +441,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } @@ -450,16 +452,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } - val test = Reg{UInt()} - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - + val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_had_data = Reg(init = Bool(false)) + val crel_was_voluntary = Reg(init = Bool(false)) + val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) + val wb_buffer = Reg{xact.data.clone} - val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) + val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, @@ -468,29 +483,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St c_gnt.payload, xact_internal.meta.coh, c_gnt.header.dst) - val is_uncached = co.messageIsUncached(xact) - val tag_match = xact_internal.tag_match - val needs_writeback = co.needsWriteback(xact_internal.meta.coh) - val is_hit = co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - //TODO: does allocate - - val c_rel_had_data = Reg(init = Bool(false)) - val c_rel_was_voluntary = Reg(init = Bool(false)) - val wb_buffer = Reg{xact.data.clone} - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, - Cat(xact_internal.meta.tag, - xact.addr(untagBits-1,blockOffBits)), + addr_wb, UInt(trackerId), wb_buffer), { case TLId => outerId }) val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, @@ -501,31 +501,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr, UInt(trackerId)) - val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, - xact.addr, + val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, + addr_wb, UInt(trackerId)) //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(needs_writeback, - inner_probe_wb, - inner_probe_cacq) + io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, + cprb_for_mwb, + cprb_for_cacq) - val grant_type = co.getGrantType(xact, xact_internal.meta.coh) + val cgnt_for_cacq = Grant(co.getGrantType(xact, xact_internal.meta.coh), + xact.client_xact_id, + UInt(trackerId), + xact.data) + val cgnt_for_cwb = Grant(crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - xact.data) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) io.data_read.valid := Bool(false) io.data_read.bits.id := UInt(trackerId) @@ -565,15 +567,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(io.meta_resp.valid) { val coh = io.meta_resp.bits.meta.coh val _tag_match = io.meta_resp.bits.tag_match - val _needs_writeback = co.needsWriteback(coh) - val _is_hit = co.isHit(xact, coh) + val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits when(_needs_probes) { - pending_probes := coh.sharers - release_count := co.dir().count(coh.sharers) - c_rel_had_data := Bool(false) - c_rel_was_voluntary := Bool(false) + val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_self = mask_incoherent & + ~(!(co.requiresSelfProbe(xact.a_type) || _needs_writeback) << init_client_id) + pending_probes := mask_self + release_count := co.dir().count(mask_self) + crel_had_data := Bool(false) + crel_was_voluntary := Bool(false) } state := Mux(_tag_match, Mux(_is_hit, @@ -585,21 +590,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - val skip = io.tile_incoherent(curr_p_id) || - ((curr_p_id === init_client_id) && - !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) - when(io.inner.probe.ready || skip) { - co.dir().pop(pending_probes, curr_p_id) + io.inner.probe.valid := !co.dir().none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir().pop(pending_probes, curr_p_id) } - when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { - c_rel_had_data := Bool(true) + crel_had_data := Bool(true) when(tag_match) { xact.data := mergeData(xact, io.inner.release.bits.payload.data) } .otherwise { @@ -607,19 +608,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } when(co.isVoluntary(c_rel.payload)) { - c_rel_was_voluntary := Bool(true) + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantType(c_rel.payload, xact_internal.meta.coh) } when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - Mux(skip, UInt(2), UInt(1)) - when(release_count === UInt(1)) { - state := Mux(tag_match, - Mux(is_hit, - Mux(c_rel_had_data, s_meta_write, s_data_read_hit), - s_outer_read), - Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) - } + release_count := release_count - UInt(1) } } + + when(crel_was_voluntary) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + crel_was_voluntary := Bool(false) + } + } + + when(release_count === UInt(0) && !crel_was_voluntary) { + state := Mux(tag_match, + Mux(is_hit, + Mux(crel_had_data, s_data_write, s_data_read_hit), + s_outer_read), + Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) + } } is(s_data_read_wb) { io.data_read.valid := Bool(true) @@ -686,6 +697,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2bb6ec01..5afcccfb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -58,6 +58,7 @@ abstract class DirectoryRepresentation(val width: Int) { def one(s: UInt): Bool def count(s: UInt): UInt def next(s: UInt): UInt + def full(s: UInt): UInt } class NullRepresentation extends DirectoryRepresentation(1) { @@ -68,6 +69,7 @@ class NullRepresentation extends DirectoryRepresentation(1) { def one(s: UInt) = Bool(false) def count(s: UInt) = UInt(0) def next(s: UInt) = UInt(0) + def full(s: UInt) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { @@ -78,6 +80,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def one(s: UInt) = PopCount(s) === UInt(1) def count(s: UInt) = PopCount(s) def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -147,6 +150,7 @@ trait UncachedTransactions { def getUncachedWriteWordAcquireType: UInt def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool + def getUncachedReadGrantType: UInt } abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) @@ -234,6 +238,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -399,6 +405,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -579,6 +587,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -756,6 +766,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -954,6 +966,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..fc4e0906 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(co.getUncachedReadGrantType, tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) From 10309849b7b7080b91281bf3cb5c95d5b5680900 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 5 Nov 2014 14:21:38 -0800 Subject: [PATCH 258/374] Remove master_xact_id from Probe and Release --- uncore/src/main/scala/tilelink.scala | 35 ++++++++++------------------ uncore/src/main/scala/uncore.scala | 8 +++---- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..6738deaf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -79,52 +79,41 @@ class Acquire extends ClientSourcedMessage val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } -object Probe -{ - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { + +object Probe { + def apply(p_type: UInt, addr: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr - prb.master_xact_id := master_xact_id prb } } class Probe extends MasterSourcedMessage - with HasPhysicalAddress - with HasMasterTransactionId { + with HasPhysicalAddress { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } -object Release -{ - def apply(r_type: UInt, addr: UInt, data: UInt): Release = { - val rel = new Release - rel.r_type := r_type - rel.addr := addr - rel.data := data - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { +object Release { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.client_xact_id := client_xact_id - rel.master_xact_id := master_xact_id - rel.data := UInt(0) - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { - val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { + apply(r_type, addr, client_xact_id, UInt(0)) + } + def apply(r_type: UInt, addr: UInt): Release = { + apply(r_type, addr, UInt(0), UInt(0)) + } } class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId - with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 37e535f8..e1f59e08 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -62,8 +62,8 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), conflict_idx) + // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -200,9 +200,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) From 35553cc0b71ffe63b1ea0b9d775e691582376db5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 16:05:25 -0800 Subject: [PATCH 259/374] NullDirectory sharers.count fix --- uncore/src/main/scala/coherence.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..9f94583a 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -61,14 +60,13 @@ abstract class DirectoryRepresentation extends Bundle { def next(dummy: Int = 0): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) +class NullRepresentation(nClients: Int) extends DirectoryRepresentation { def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) + def count(dummy: Int = 0) = UInt(nClients) def next(dummy: Int = 0) = UInt(0) } From 82155f333e96bfb5eaa3b3b25ff7184609e39a17 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 17:36:55 -0800 Subject: [PATCH 260/374] Major tilelink revision for uncached message types --- uncore/src/main/scala/coherence.scala | 566 ++++++++++---------------- uncore/src/main/scala/consts.scala | 2 + uncore/src/main/scala/htif.scala | 6 +- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 179 +++++--- uncore/src/main/scala/uncore.scala | 31 +- 6 files changed, 351 insertions(+), 435 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9f94583a..a2af20cb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -120,59 +120,48 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt - def messageHasData (rel: SourcedMessage): Bool - def messageUpdatesDataArray (reply: Grant): Bool - def messageIsUncached(acq: Acquire): Bool + def messageHasData(rel: SourcedMessage): Bool + def messageUpdatesDataArray(g: Grant): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterRead(a_type: UInt): Bool - def requiresOuterWrite(a_type: UInt): Bool - def requiresSelfProbe(a_type: UInt): Bool - def requiresAckForGrant(g_type: UInt): Bool - def requiresAckForRelease(r_type: UInt): Bool + def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresSelfProbe(a: Acquire): Bool + def requiresAckForGrant(g: Grant): Bool + def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) + } } -trait UncachedTransactions { - def getUncachedReadAcquireType: UInt - def getUncachedWriteAcquireType: UInt - def getUncachedReadWordAcquireType: UInt - def getUncachedWriteWordAcquireType: UInt - def getUncachedAtomicAcquireType: UInt - def isUncachedReadTransaction(acq: Acquire): Bool -} - -abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) - with UncachedTransactions - -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 - def nAcquireTypes = 6 + def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 - def nGrantTypes = 7 + def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid @@ -197,15 +186,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit M_CLN -> clientValid )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> clientValid, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -225,14 +207,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive @@ -251,80 +227,61 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } + def getGrantType(a: Acquire, m: MasterMetadata): UInt = + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 8 + def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid @@ -332,7 +289,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -355,16 +312,9 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -387,14 +337,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) @@ -419,83 +363,62 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusiveClean -> grantReadExclusive, - acquireReadExclusiveDirty -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusiveClean -> probeInvalidate, - acquireReadExclusiveDirty -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), @@ -508,7 +431,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -530,16 +453,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> clientExclusiveDirty, - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(incoming.g_type === grantReadShared, clientShared, + clientExclusiveDirty)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -564,14 +480,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -596,79 +506,69 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeInvalidate, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), @@ -681,7 +581,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -704,16 +604,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -738,14 +635,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -770,83 +661,68 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 3 - def nAcquireTypes = 8 + def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 - def nGrantTypes = 9 + def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, m: ClientMetadata): Bool = { @@ -859,7 +735,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -883,22 +759,18 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -935,14 +807,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) @@ -969,60 +835,50 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + MuxLookup(a.a_type, grantReadShared, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? + ))) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index a402e7db..fe647d07 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,9 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") + val MT_CB = Bits("b111") // cache block + val NUM_XA_OPS = 9 val M_SZ = 5 val M_X = Bits("b?????"); val M_XRD = Bits("b00000"); // int load diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index edcd66cb..405380f2 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -135,7 +135,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } io.mem.grant.ready := Bool(true) @@ -193,8 +193,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), - Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + UncachedWrite(init_addr, UInt(0)), + UncachedRead(init_addr)) io.mem.acquire.valid := acq_q.io.deq.valid acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..02e651b8 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(Bool(true), UInt(0), tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6738deaf..3b1bf26b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -2,65 +2,89 @@ package uncore import Chisel._ +import scala.math.max case object TLId extends Field[String] -case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] -case object TLWriteMaskBits extends Field[Int] -case object TLWordAddrBits extends Field[Int] -case object TLAtomicOpBits extends Field[Int] -trait HasPhysicalAddress extends Bundle { - val addr = UInt(width = params(TLAddrBits)) +abstract trait TileLinkParameters extends UsesParameters { + val tlAddrBits = params(TLAddrBits) + val tlClientXactIdBits = params(TLClientXactIdBits) + val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlDataBits = params(TLDataBits) + val tlWriteMaskBits = tlDataBits/8 + val tlSubblockAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlUncachedOperandSizeBits = MT_SZ + val tlSubblockUnionBits = max(tlWriteMaskBits, + (tlSubblockAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = params(TLClientXactIdBits)) +class TLBundle extends Bundle with TileLinkParameters + +trait HasPhysicalAddress extends TLBundle { + val addr = UInt(width = tlAddrBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = params(TLMasterXactIdBits)) +trait HasClientTransactionId extends TLBundle { + val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasTileLinkData extends Bundle { - val data = Bits(width = params(TLDataBits)) +trait HasMasterTransactionId extends TLBundle { + val master_xact_id = Bits(width = tlMasterXactIdBits) } -trait SourcedMessage extends Bundle +trait HasTileLinkData extends TLBundle { + val data = UInt(width = tlDataBits) +} + +trait SourcedMessage extends TLBundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -object Acquire -{ - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { +class Acquire extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val uncached = Bool() + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val subblock = Bits(width = tlSubblockUnionBits) + val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits + val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) + def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) +} + +object Acquire { + val nUncachedAcquireTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite + def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = new Acquire + acq.uncached := Bool(false) acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.data := Bits(0) - acq.write_mask := Bits(0) - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) - acq - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id) acq.data := data + acq.subblock := UInt(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.write_mask := write_mask - acq - } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.subword_addr := subword_addr - acq.atomic_opcode := atomic_opcode - acq + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { + apply(a_type, addr, client_xact_id, UInt(0)) } def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -69,16 +93,45 @@ object Acquire } } -class Acquire extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) - val write_mask = Bits(width = params(TLWriteMaskBits)) - val subword_addr = Bits(width = params(TLWordAddrBits)) - val atomic_opcode = Bits(width = params(TLAtomicOpBits)) +object UncachedRead { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) + acq.uncached := Bool(true) + acq.subblock := Cat(subblock_addr, operand_sz) + acq + } + def apply(addr: UInt, client_xact_id: UInt): Acquire = { + apply(addr, client_xact_id, UInt(0), MT_CB) + } + def apply(addr: UInt): Acquire = { + apply(addr, UInt(0), UInt(0), MT_CB) + } } +object UncachedWrite { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := write_mask + acq + } + def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(addr, client_xact_id, SInt(-1), data) + } + def apply(addr: UInt, data: UInt): Acquire = { + apply(addr, UInt(0), data) + } +} + +object UncachedAtomic { + def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, + subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq + } +} object Probe { def apply(p_type: UInt, addr: UInt) = { @@ -118,28 +171,34 @@ class Release extends ClientSourcedMessage val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } -object Grant -{ - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - val gnt = new Grant - gnt.g_type := g_type - gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id - gnt.data := UInt(0) - gnt - } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { - val gnt = apply(g_type, client_xact_id, master_xact_id) - gnt.data := data - gnt - } -} - class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + val uncached = Bool() + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) +} + +object Grant { + val nUncachedGrantTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + val gnt = new Grant + gnt.uncached := uncached + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := data + gnt + } + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + } } class Finish extends ClientSourcedMessage with HasMasterTransactionId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e1f59e08..6ea8cd25 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -122,7 +122,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), xact.data), @@ -133,7 +133,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), + io.inner.grant.bits.payload := Grant(Bool(false), + co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -169,15 +170,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) + val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -202,11 +201,11 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.bits.header.dst := curr_p_id io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, + io.inner.grant.bits.payload := Grant(xact.uncached, + co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) @@ -218,7 +217,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src @@ -266,7 +265,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_mem_write) { @@ -279,7 +278,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_make_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete From cb7e7125993c001c08d5159aa1e4d8bda3837c00 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Nov 2014 12:55:07 -0800 Subject: [PATCH 261/374] Added uncached write data queue to coherence hub --- uncore/src/main/scala/tilelink.scala | 2 +- uncore/src/main/scala/uncore.scala | 59 +++++++++++++++++++++------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b1bf26b..ec4c15df 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,7 +16,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) - val tlWriteMaskBits = tlDataBits/8 + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) val tlUncachedOperandSizeBits = MT_SZ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6ea8cd25..858d603c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -5,14 +5,17 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) - val nReleaseTransactors = params(NReleaseTransactors) + val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) + val sdqDepth = params(L2StoreDataQueueDepth) + val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { + // Queue to store impending UncachedWrite data + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} + when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId))) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -67,12 +79,19 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } + when(voluntary && release.fire()) { + vwbdq(release_idx) := release.bits.payload.data + } + // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) + { case TLId => outerId; case TLDataBits => sdqIdBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) + val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) + val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) + val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) + val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq + io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), + Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } } @@ -112,10 +144,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -140,11 +171,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute switch (state) { is(s_idle) { - io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + io.inner.release.ready := Bool(true) + xact := c_rel.payload + init_client_id := c_rel.header.src + state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) } } is(s_mem) { @@ -173,7 +204,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) @@ -208,7 +239,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + UInt(0)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 05b5188ad97607b5534b60a56513acc8d853fcea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 19 Nov 2014 15:55:25 -0800 Subject: [PATCH 262/374] meta and data bundle refactor --- uncore/src/main/scala/cache.scala | 198 +++++++++++++++--------------- 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 10d00210..ee375784 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -138,12 +138,19 @@ class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState +trait HasL2MetaReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2MetaReadReq) + val resp = Valid(new L2MetaResp).flip +} + +trait HasL2MetaWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2MetaWriteReq) +} + +class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaWriteIO + class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } + val io = new L2MetaRWIO().flip val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read @@ -189,12 +196,19 @@ class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { val data = Bits(width = tlDataBits) } +trait HasL2DataReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2DataReadReq) + val resp = Valid(new L2DataResp).flip +} + +trait HasL2DataWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2DataWriteReq) +} + +class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO + class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } + val io = new L2DataRWIO().flip val waddr = io.write.bits.addr val raddr = io.read.bits.addr @@ -226,12 +240,8 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp + tshrfile.io.meta <> meta.io + tshrfile.io.data <> data.io io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -242,12 +252,8 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val meta = new L2MetaRWIO + val data = new L2DataRWIO } // Wiring helper funcs @@ -323,12 +329,12 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac io.outer <> outer_arb.io.out // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) } @@ -340,12 +346,8 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val data = new L2DataRWIO + val meta = new L2MetaRWIO } val c_acq = io.inner.acquire.bits @@ -386,23 +388,23 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact.client_xact_id, UInt(trackerId)) - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_internal.meta.coh, init_client_id) @@ -416,24 +418,24 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := Mux(io.meta_resp.bits.tag_match, + when(io.meta.resp.valid) { + xact_internal := io.meta.resp.bits + state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { state := s_meta_write } + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) @@ -522,26 +524,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := next_coh_on_grant + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_internal.way_en + io.data.read.bits.addr := xact.addr + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -553,17 +555,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - val coh = io.meta_resp.bits.meta.coh - val _tag_match = io.meta_resp.bits.tag_match + when(io.meta.resp.valid) { + val coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta_resp.bits + xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & @@ -626,14 +628,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_wb) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_wb } } is(s_data_resp_wb) { - when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits.data + when(io.data.resp.valid) { + wb_buffer := io.data.resp.bits.data state := s_outer_write_wb } } @@ -661,26 +663,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_hit) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_hit } } is(s_data_resp_hit) { - when(io.data_resp.valid) { - xact.data := mergeData(xact, io.data_resp.bits.data) + when(io.data.resp.valid) { + xact.data := mergeData(xact, io.data.resp.bits.data) state := s_meta_write } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) From 404773eb9f1f29277b205eb4ace622ca67eb0d9d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 3 Dec 2014 14:22:39 -0800 Subject: [PATCH 263/374] fix wb bug --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ee375784..0a717604 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -462,6 +462,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Reg{xact.data.clone} + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) @@ -476,8 +477,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && + (state != s_idle) //TODO: Also indexes? val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, @@ -488,7 +490,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_internal.meta.coh, c_gnt.header.dst) - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), From 2f733a60db1d564f2da11632acc6e3308910460c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 02:57:44 -0800 Subject: [PATCH 264/374] Begin adding TLDataBeats to uncore --- uncore/src/main/scala/network.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index d3f4a9e8..a34e91c1 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -25,13 +25,13 @@ class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { abstract class PhysicalNetwork extends Module -class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index a5066f39..1e535507 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -10,12 +10,14 @@ case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] +case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) From 3026c46a9c81709bd1c49e1f9737af1095cf918c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 03:02:20 -0800 Subject: [PATCH 265/374] Finish adding TLDataBeats to uncore & hub --- uncore/src/main/scala/cache.scala | 4 +- uncore/src/main/scala/coherence.scala | 6 - uncore/src/main/scala/htif.scala | 33 +++-- uncore/src/main/scala/memserdes.scala | 133 ++++++++++++------ uncore/src/main/scala/tilelink.scala | 18 +-- uncore/src/main/scala/uncore.scala | 193 ++++++++++++++++---------- uncore/src/main/scala/util.scala | 88 ++++++++++++ 7 files changed, 325 insertions(+), 150 deletions(-) create mode 100644 uncore/src/main/scala/util.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0a717604..b96aab70 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,7 +30,8 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCycles = params(TLDataBits)/rowBits + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +100,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters - with TileLinkParameters abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 57545fda..280e20ad 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,12 +3,6 @@ package uncore import Chisel._ -object MuxBundle { - def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) - } -} - abstract class CoherenceMetadata extends Bundle object ClientMetadata { diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 405380f2..378ca3b1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -13,6 +13,7 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) + val dataBeats = params(TLDataBeats) val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) @@ -71,7 +72,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + dataBits + val long_request_bits = short_request_bits + dataBits*dataBeats require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -150,12 +151,13 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state_tx))) } - val acq_q = Module(new Queue(new Acquire, 1)) - when (state === state_mem_wreq && acq_q.io.enq.ready) { - state := state_mem_wresp + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) + when (state === state_mem_wreq) { + when (cnt_done) { state := state_mem_wresp } } - when (state === state_mem_rreq && acq_q.io.enq.ready) { - state := state_mem_rresp + when (state === state_mem_rreq) { + when(io.mem.acquire.ready) { state := state_mem_rresp } } when (state === state_mem_wresp) { when (mem_acked) { @@ -164,10 +166,10 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } when (state === state_mem_rresp) { - when (io.mem.grant.valid) { + when (cnt_done) { state := state_mem_finish + mem_acked := Bool(false) } - mem_acked := Bool(false) } when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) @@ -182,22 +184,19 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: Bits = null + var mem_req_data: UInt = null for (i <- 0 until dataBits/short_request_bits) { - val idx = UInt(i, log2Up(dataBits/short_request_bits)) + val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) - acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, UInt(0)), + io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq + io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + UncachedWrite(init_addr, mem_req_data), UncachedRead(init_addr)) - io.mem.acquire.valid := acq_q.io.deq.valid - acq_q.io.deq.ready := io.mem.acquire.ready - io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module @@ -255,7 +254,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index ea41256e..d6133fc0 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,81 +209,122 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val co = params(TLCoherence) - val tbits = params(MIFTagBits) - val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBeats) - require(params(TLDataBits) == dbits*dbeats) + val mifTagBits = params(MIFTagBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) + val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + // Decompose outgoing TL Acquires into MemIO cmd and data val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = dbeats - val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + io.uncached.acquire.ready := Bool(false) + io.uncached.grant.valid := Bool(false) + io.mem.resp.ready := Bool(false) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + + val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val buf_out = Reg(Bits()) + val tl_done_out = Reg(init=Bool(false)) + val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) - val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) - val active_in = Reg(init=Bool(false)) - val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = tbits)) - - // Decompose outgoing TL Acquires into MemIO cmd and data - when(!active_out && io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - cnt_out := UInt(0) - buf_out := io.uncached.acquire.bits.payload.data - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr - has_data := co.messageHasData(io.uncached.acquire.bits.payload) + when(!active_out){ + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := acq_has_data + tl_done_out := tl_wrap_out + mif_done_out := Bool(false) + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } } when(active_out) { + when(!cmd_sent_out) { + mem_cmd_q.io.enq.valid := Bool(true) + } + when(has_data) { + when(!tl_done_out) { + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + } + } when(mem_cmd_q.io.enq.fire()) { cmd_sent_out := Bool(true) } - when(mem_data_q.io.enq.fire()) { - cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(dbits) - } - when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(cmd_sent_out && (!has_data || mif_done_out)) { active_out := Bool(false) } } - io.uncached.acquire.ready := !active_out - mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out mem_cmd_q.io.enq.bits.rw := has_data mem_cmd_q.io.enq.bits.tag := tag_out mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) - mem_data_q.io.enq.bits.data := buf_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data io.mem.req_cmd <> mem_cmd_q.io.deq io.mem.req_data <> mem_data_q.io.deq // Aggregate incoming MemIO responses into TL Grants - io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) - io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), buf_in) - when(!active_in && io.mem.resp.valid) { - active_in := Bool(true) - cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) - tag_in := io.mem.resp.bits.tag + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) + val active_in = Reg(init=Bool(false)) + val mif_done_in = Reg(init=Bool(false)) + val tag_in = Reg(UInt(width = mifTagBits)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + + when(!active_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data + } } + when(active_in) { - when(io.uncached.grant.fire()) { - active_in := Bool(false) - } - when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) - cnt_in := cnt_in + UInt(1) + io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } } + + io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), + tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -390,8 +431,8 @@ class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) val a = Module(new MemIOUncachedTileLinkIOConverter(2)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) a.io.uncached <> io.uncached - b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1e535507..678efb6a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -26,9 +26,11 @@ abstract trait TileLinkParameters extends UsesParameters { (tlSubblockAddrBits + tlUncachedOperandSizeBits + tlAtomicOpcodeBits)) + val co = params(TLCoherence) } -class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLModule extends Module with TileLinkParameters trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) @@ -55,7 +57,7 @@ class Acquire extends ClientSourcedMessage with HasClientTransactionId with HasTileLinkData { val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff @@ -147,7 +149,7 @@ object Probe { class Probe extends MasterSourcedMessage with HasPhysicalAddress { - val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + val p_type = UInt(width = co.probeTypeWidth) def is(t: UInt) = p_type === t } @@ -172,7 +174,7 @@ class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + val r_type = UInt(width = co.releaseTypeWidth) def is(t: UInt) = r_type === t } @@ -181,7 +183,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t } @@ -221,7 +223,7 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -abstract class TileLinkArbiterLike(val arbN: Int) extends Module { +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -232,8 +234,8 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends Module { def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) - val arb = Module(new RRArbiter(out.bits.clone, arbN)) + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { arb.valid := req.valid diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 858d603c..69b13800 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -8,14 +8,16 @@ case object NAcquireTransactors extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] -abstract trait CoherenceAgentParameters extends UsesParameters { - val co = params(TLCoherence) +abstract trait CoherenceAgentParameters extends UsesParameters + with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) - val sdqDepth = params(L2StoreDataQueueDepth) - val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 + val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + + log2Ceil(tlDataBeats) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,75 +29,93 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } +class DataQueueLocation extends Bundle with CoherenceAgentParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { - // Queue to store impending UncachedWrite data - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) - val sdq_rdy = !sdq_val.andR - val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) - val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} - when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle acquire transaction initiation + // Queue to store impending UncachedWrite data val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits t.acquire.valid := alloc_arb.io.in(i).ready } acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Handle releases, which might be voluntary and might have data + // Queue to store impending Voluntary Release data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) + val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width + val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} val release_idx = Mux(voluntary, UInt(0), conflict_idx) - // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } - when(voluntary && release.fire()) { - vwbdq(release_idx) := release.bits.payload.data - } + // Wire probe requests to clients + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Free finished transactions + // Wire finished transaction acks val ack = io.inner.finish trackerList.map(_.io.inner.finish.valid := ack.valid) trackerList.map(_.io.inner.finish.bits := ack.bits) @@ -103,27 +123,28 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => sdqIdBits }) + { case TLId => outerId; case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) - val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) - val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) - val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) - val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq - io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), - Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + co.messageHasData(io.outer.acquire.bits.payload) && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out // Update SDQ valid bits when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) } } abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) + val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) @@ -137,13 +158,20 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && @@ -156,7 +184,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), - xact.data), + data_ptrs(outer_data_cnt)), { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -169,18 +197,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact.client_xact_id, UInt(trackerId)) + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_ptrs(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { + io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - io.inner.release.ready := Bool(true) xact := c_rel.payload init_client_id := c_rel.header.src - state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) + data_ptrs(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) + state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + is(s_outer) { + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + when(outer_data_done) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) @@ -195,6 +233,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact = Reg{ new Acquire } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) @@ -202,23 +246,21 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.acquire.valid := Bool(false) @@ -244,6 +286,14 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + when(collect_inner_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_ptrs(inner_data_cnt) := c_acq.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -252,14 +302,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src + data_ptrs(UInt(0)) := c_acq.payload.data + collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) } } is(s_probe) { @@ -276,15 +325,17 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + when(outer_data_done) { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } } .otherwise { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -300,9 +351,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := Bool(true) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { + when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala new file mode 100644 index 00000000..c28778aa --- /dev/null +++ b/uncore/src/main/scala/util.scala @@ -0,0 +1,88 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import scala.math._ + +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) + } +} + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { + val io = new Bundle { + val in = Decoupled(gen.clone).flip + val out = Decoupled(gen.clone) + val cnt = UInt(OUTPUT, log2Up(n)) + val done = Bool(OUTPUT) + } + val narrowWidth = io.in.bits.payload.data.getWidth / n + require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + + if(n == 1) { + io.in <> io.out + io.cnt := UInt(width = 0) + io.done := Bool(true) + } else { + val cnt = Reg(init=UInt(0, width = log2Up(n))) + val wrap = cnt === UInt(n-1) + val rbits = Reg(init=io.in.bits) + val active = Reg(init=Bool(false)) + + val shifter = Vec.fill(n){Bits(width = narrowWidth)} + (0 until n).foreach { + i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + } + + io.done := Bool(false) + io.cnt := cnt + io.in.ready := !active + io.out.valid := active || io.in.valid + io.out.bits := io.in.bits + when(!active && io.in.valid) { + when(doSer(io.in.bits.payload)) { + cnt := Mux(io.out.ready, UInt(1), UInt(0)) + rbits := io.in.bits + active := Bool(true) + } + io.done := !doSer(io.in.bits.payload) + } + when(active) { + io.out.bits := rbits + io.out.bits.payload.data := shifter(cnt) + when(io.out.ready) { + cnt := cnt + UInt(1) + when(wrap) { + cnt := UInt(0) + io.done := Bool(true) + active := Bool(false) + } + } + } + } +} From 424df2368f6c93f34800f61bf7789d84d7762663 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 01:11:08 -0800 Subject: [PATCH 266/374] 1R/W L2 data array? Add TLDataBeats to new LLC; all bmarks pass --- uncore/src/main/scala/cache.scala | 335 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 26 +- uncore/src/main/scala/tilelink.scala | 9 +- uncore/src/main/scala/uncore.scala | 142 ++++++----- 4 files changed, 308 insertions(+), 204 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b96aab70..97a0c9a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -134,6 +134,7 @@ class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id { override def clone = new L2MetaWriteReq().asInstanceOf[this.type] } + class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -156,10 +157,10 @@ class L2MetadataArray extends L2HellaCacheModule { meta.io.read <> io.read meta.io.write <> io.write - val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -220,11 +221,11 @@ class L2DataArray extends L2HellaCacheModule { } array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + io.resp.valid := ShiftRegister(io.read.fire(), 1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - io.read.ready := Bool(true) + io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? io.write.ready := Bool(true) } @@ -233,7 +234,6 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends require(isPow2(nSets)) require(isPow2(nWays)) - require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) @@ -257,8 +257,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + def doOutputArbitration[T <: Data](out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } @@ -282,7 +285,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner @@ -293,9 +295,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - // Handle releases, which might be voluntary and might have data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) @@ -310,10 +309,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions on ack + // Wire finished transaction acks val finish = io.inner.finish val finish_idx = finish.bits.payload.master_xact_id trackerList.zipWithIndex.map { case (t, i) => @@ -322,20 +318,26 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac trackerList.map(_.io.inner.finish.bits := finish.bits) finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - // Arbitrate for the outer memory port + // Wire probe requests to clients + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + + // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out - // Local memory + // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) - } @@ -356,21 +358,34 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - def mergeData(acq: Acquire, data: UInt): UInt = { - //TODO wite mask - Mux(co.messageHasData(acq), acq.data, data) + def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { + //TODO apply acq's write mask + Mux(co.messageHasData(acq), old_data, new_data) } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (local_data_cnt, local_data_done) = + Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -382,38 +397,50 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.finish.ready := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh), - xact.client_xact_id, + co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), + xact_client_xact_id, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, - xact_internal.meta.coh, - init_client_id) + xact_meta.coh, + xact_src) + + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_data(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) state := s_meta_read } } @@ -423,15 +450,18 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta.resp.valid) { - xact_internal := io.meta.resp.bits + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { state := s_meta_write } + io.data.write.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (local_data_cnt < inner_data_cnt)) + when(local_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -454,72 +484,100 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val crel_had_data = Reg(init = Bool(false)) val crel_was_voluntary = Reg(init = Bool(false)) val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Reg{xact.data.clone} - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) + val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + + val collect_cacq_data = Reg(init=Bool(false)) + //TODO: zero width wires + val (cacq_data_cnt, cacq_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (outer_data_resp_cnt, outer_data_resp_done) = + Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - val is_uncached = xact.uncached - val tag_match = xact_internal.tag_match - val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) - val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact, xact_internal.meta.coh) - //TODO: does allocate + val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) + val needs_probes = co.requiresProbes(xact, xact_meta.coh) + //TODO: uncached does or does not allocate - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes + //TODO: Are there any races between lines with the same idx? + //TODO: Allow hit under miss for stores + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + (state != s_idle) && + !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && - (state != s_idle) //TODO: Also indexes? + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state != s_idle) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, - xact_internal.meta.coh, + xact_meta.coh, c_rel.header.src) val next_coh_on_grant = co.masterMetadataOnGrant( c_gnt.payload, - xact_internal.meta.coh, + xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), + val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact.addr, UInt(trackerId)), { case TLId => outerId }) + val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr) - val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, addr_wb) - //TODO inner_probe_mprb + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) + //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, - cprb_for_mwb, + io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, + cprb_for_mvwb, cprb_for_cacq) - val cgnt_for_cacq = Grant(xact.uncached, co.getGrantType(xact, xact_internal.meta.coh), - xact.client_xact_id, + val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, UInt(trackerId), - xact.data) - val cgnt_for_cwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) + xact_data(cgnt_data_cnt)) + val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -527,31 +585,45 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_internal.way_en - io.data.read.bits.addr := xact.addr + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := next_coh_on_grant + when(collect_cacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_data(cacq_data_cnt) := c_acq.payload.data + } + when(cacq_data_done) { collect_cacq_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src + collect_cacq_data := co.messageHasData(c_acq.payload) state := s_meta_read } } @@ -561,21 +633,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_meta_resp) { when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << init_client_id) + ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self release_count := co.dir().count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) - } + } state := Mux(_tag_match, Mux(_is_hit, Mux(_needs_probes, s_probe, s_data_read_hit), @@ -594,25 +668,41 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_internal.meta.coh := next_coh_on_release + xact_meta.coh := next_coh_on_release + + // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(tag_match) { - xact.data := mergeData(xact, io.inner.release.bits.payload.data) - } .otherwise { - wb_buffer := io.inner.release.bits.payload.data + when(xact_tag_match) { // Hit, so merge new write with released data + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + io.inner.release.bits.payload.data) + } .otherwise { // Miss, we're voluntarily evicting this data + wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data } } + + // Voluntary releases don't count against the release countdown + // because we will get a further release ack from that client in + // response to our probe. We don't send a grant acknowledging + // a writeback or decrement release_count until we've received + // all the data beats. when(co.isVoluntary(c_rel.payload)) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh) - } - when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - UInt(1) + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) + } + } .otherwise { + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } } } + // If we saw a voluntary writeback, we need to send an extra grant + // to acknowledge it. when(crel_was_voluntary) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { @@ -621,7 +711,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(tag_match, + state := Mux(xact_tag_match, Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read_hit), s_outer_read), @@ -630,20 +720,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_read_wb) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_wb + when(local_data_read_done) { state := s_data_resp_wb } + when(io.data.resp.valid) { + wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } } is(s_data_resp_wb) { - when(io.data.resp.valid) { - wb_buffer := io.data.resp.bits.data - state := s_outer_write_wb - } + when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write_wb } } is(s_outer_write_wb) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_wb - when(io.outer.acquire.ready) { + when(outer_data_write_done) { state := s_outer_read } } @@ -657,27 +746,35 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_outer_resp) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - xact.data := mergeData(xact, io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_internal.meta.coh - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + //TODO make sure cacq data is actually present before merging + xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), + io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_meta.coh + when(outer_data_resp_done) { + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) + } } } is(s_data_read_hit) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_hit + when(io.data.resp.valid) { + //TODO make sure cacq data is actually present before merging + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_read_done) { state := s_data_resp_hit } } is(s_data_resp_hit) { when(io.data.resp.valid) { - xact.data := mergeData(xact, io.data.resp.bits.data) - state := s_meta_write + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_resp_done) { state := s_meta_write } } is(s_data_write) { io.data.write.valid := Bool(true) - when(io.data.write.ready) { + when(local_data_write_done) { state := s_meta_write } } @@ -687,7 +784,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { + when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { state := Mux(co.requiresAckForGrant(c_gnt.payload), s_busy, s_idle) } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 280e20ad..9d6d768b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -205,9 +205,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -339,9 +337,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -485,10 +481,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -644,11 +637,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -817,12 +806,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached, - g.is(grantReadMigratory) -> cached - )) + Mux(g.uncached, uncached, cached) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 678efb6a..45c6fa59 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -78,16 +78,19 @@ object Acquire { def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { val acq = new Acquire - acq.uncached := Bool(false) + acq.uncached := uncached acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.data := data - acq.subblock := UInt(0) + acq.subblock := subblock acq } + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) + } def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { apply(a_type, addr, client_xact_id, UInt(0)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 69b13800..fcdbdd9f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,20 +15,10 @@ abstract trait CoherenceAgentParameters extends UsesParameters val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + - log2Ceil(tlDataBeats) + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } -} - class DataQueueLocation extends Bundle with CoherenceAgentParameters { val idx = UInt(width = dqIdxBits) val loc = UInt(width = log2Ceil(nDataQueueLocations)) @@ -42,6 +32,15 @@ object DataQueueLocation { } } +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module + with CoherenceAgentParameters { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { @@ -103,6 +102,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + // Wire probe requests to clients val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.inner.probe <> probe_arb.io.out @@ -115,12 +120,6 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), { case TLId => outerId; case TLDataBits => internalDataBits }) @@ -164,9 +163,13 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) @@ -174,33 +177,35 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact.addr, - UInt(trackerId), - data_ptrs(outer_data_cnt)), - { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(UncachedWrite( + xact_addr, + UInt(trackerId), + xact_data(outer_data_cnt)), + { case TLId => outerId }) + when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - data_ptrs(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := c_rel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -209,15 +214,18 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - data_ptrs(UInt(0)) := c_rel.payload.data + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data collect_inner_data := co.messageHasData(c_rel.payload) state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) when(outer_data_done) { state := s_ack } } is(s_ack) { @@ -230,28 +238,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) val (outer_data_cnt, outer_data_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), - { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -260,26 +268,34 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + (state != s_idle) && + !collect_inner_data + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state != s_idle) + + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(xact.uncached, + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -289,7 +305,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - data_ptrs(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := c_acq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -300,9 +316,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - data_ptrs(UInt(0)) := c_acq.payload.data + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) @@ -319,12 +339,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back + io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) when(outer_data_done) { release_count := release_count - UInt(1) when(release_count === UInt(1)) { @@ -334,7 +354,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } .otherwise { - io.inner.release.ready := Bool(true) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -351,7 +370,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) From 6a8b66231ca484cd40269a8aa8170c0ef318b5fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 12:07:04 -0800 Subject: [PATCH 267/374] Add uncached->cached tilelink converter --- uncore/src/main/scala/tilelink.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45c6fa59..4e5ac961 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -226,6 +226,25 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} +object TileLinkIOWrapper { + def apply[T <: Data](uncached: UncachedTileLinkIO) = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> uncached + conv.io.out + } +} + abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId From d04da83f96b58c7957483059ea7508d4d38925cd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 15 Dec 2014 17:36:17 -0800 Subject: [PATCH 268/374] Make data RAMs 1RW instead of 1R1W --- uncore/src/main/scala/cache.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 97a0c9a5..09d04521 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -216,10 +216,13 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) + val reg_raddr = Reg(UInt()) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { + reg_raddr := raddr } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(reg_raddr) } io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) From ab39cbb15dbe4068f96882d7ab1923ad51c524e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Dec 2014 19:23:13 -0800 Subject: [PATCH 269/374] cleanup DirectoryRepresentation and coherence params --- uncore/src/main/scala/cache.scala | 14 +-- uncore/src/main/scala/coherence.scala | 145 ++++++++++++-------------- uncore/src/main/scala/uncore.scala | 17 ++- 3 files changed, 86 insertions(+), 90 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 09d04521..08533ef9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -123,7 +123,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone + val coh = new MasterMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -524,8 +524,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir().flush) - val curr_p_id = co.dir().next(pending_probes) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) @@ -645,11 +645,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self - release_count := co.dir().count(mask_self) + release_count := co.dir.count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) } @@ -663,9 +663,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - io.inner.probe.valid := !co.dir().none(pending_probes) + io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { - pending_probes := co.dir().pop(pending_probes, curr_p_id) + pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9d6d768b..d7bbd25f 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,46 +3,35 @@ package uncore import Chisel._ -abstract class CoherenceMetadata extends Bundle +abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata().asInstanceOf[this.type] +} object ClientMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy) = { + def apply(state: UInt) = { val m = new ClientMetadata m.state := state m } } -class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata()(c).asInstanceOf[this.type] -} +class MasterMetadata extends CoherenceMetadata { + val state = UInt(width = co.masterStateWidth) + val sharers = UInt(width = co.dir.width) + override def clone = new MasterMetadata().asInstanceOf[this.type] +} object MasterMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt) = { val m = new MasterMetadata m.state := state - m.sharers := c.dir().flush - m - } - def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { - val m = apply(state) m.sharers := sharers m } + def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) } -class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.masterStateWidth) - val sharers = UInt(width = c.dir().width) - override def clone = new MasterMetadata()(c).asInstanceOf[this.type] -} -/* -class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { - val cstate = UInt(width = outer.clientStateWidth) - val mstate = UInt(width = inner.masterStateWidth) - val sharers = inner.dir.sharers.clone -} -*/ abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt @@ -77,7 +66,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } -abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -143,7 +132,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { } } -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 1 @@ -184,18 +173,18 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientValid - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -203,8 +192,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -263,11 +252,11 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -307,27 +296,27 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientExclusiveClean - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean)))(this) + clientExclusiveClean))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientExclusiveClean, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -335,8 +324,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -401,12 +390,12 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -452,26 +441,26 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty)))(this) + clientExclusiveDirty))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -479,8 +468,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -525,7 +514,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -552,12 +541,12 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 2 def nAcquireTypes = 2 @@ -603,13 +592,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -618,16 +607,16 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), grantReadExclusiveAck -> clientExclusiveDirty - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -635,8 +624,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -681,7 +670,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -708,12 +697,12 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 2 def nAcquireTypes = 3 @@ -760,12 +749,12 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + clientMigratoryClean -> clientMigratoryDirty)), m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -779,7 +768,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> clientMigratoryDirty, acquireReadExclusive -> clientMigratoryDirty, acquireReadShared -> clientMigratoryClean)) - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -791,10 +780,10 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientSharedByTwo -> clientShared, clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -804,8 +793,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -853,7 +842,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) @@ -883,7 +872,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index fcdbdd9f..411d9517 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -4,16 +4,22 @@ package uncore import Chisel._ case object NReleaseTransactors extends Field[Int] +case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object NIncoherentClients extends Field[Int] +case object NCoherentClients extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] -case object NClients extends Field[Int] +case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CoherenceAgentParameters extends UsesParameters with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nClients = params(NClients) + val nCoherentClients = params(NCoherentClients) + val nIncoherentClients = params(NIncoherentClients) + val nClients = nCoherentClients + nIncoherentClients val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases @@ -41,7 +47,7 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } -class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends +class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { val internalDataBits = new DataQueueLocation().getWidth @@ -143,11 +149,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) + val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) + val nClients = params(NCoherentClients) + params(NIncoherentClients) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, params(NClients)) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } From bfcfc3fe18ba88c45fec946324ac3e8bcff1c417 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 17 Dec 2014 14:28:14 -0800 Subject: [PATCH 270/374] refactor cache params --- uncore/src/main/scala/cache.scala | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08533ef9..a63fafff 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,8 +30,6 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +97,12 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { } abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters + with CoherenceAgentParameters { + val idxMSB = idxBits-1 + val idxLSB = 0 + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) +} abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters @@ -415,13 +418,13 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) @@ -505,7 +508,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires @@ -535,7 +538,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || @@ -598,13 +601,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_grant when(collect_cacq_data) { From d121af7f942e3d1f25084e9d3a7000ef3f03fd77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 18 Dec 2014 17:12:29 -0800 Subject: [PATCH 271/374] Simplify release handling --- uncore/src/main/scala/cache.scala | 22 +++++++++------------- uncore/src/main/scala/uncore.scala | 17 +++++++---------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a63fafff..e3a8db22 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -303,17 +303,13 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle releases, which might be voluntary and might have data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -353,7 +349,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO } @@ -391,8 +387,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -541,9 +536,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data - io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && - (state != s_idle) + io.has_release_match := !co.isVoluntary(c_rel.payload) && + (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state === s_probe) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 411d9517..d98bf980 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -94,19 +94,16 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } // Handle releases, which might be voluntary and might have data - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, UInt(0), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits t.release.bits.payload.data := (if (i < nReleaseTransactors) DataQueueLocation(rel_data_cnt, inVolWBQueue) else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val ack = io.inner.finish @@ -156,7 +153,7 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) } val c_acq = io.inner.acquire.bits @@ -184,8 +181,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -278,7 +274,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + !co.isVoluntary(c_rel.payload) && (state != s_idle) val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), From f234fe65ce4eb88d09fd10c0707a6bcf2d6f1269 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 03:03:53 -0800 Subject: [PATCH 272/374] Initial verison of L2WritebackUnit, passes MiT2 bmark tests --- uncore/src/main/scala/cache.scala | 377 ++++++++++++++++---------- uncore/src/main/scala/coherence.scala | 21 +- 2 files changed, 253 insertions(+), 145 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3a8db22..3cbc8967 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -108,7 +108,7 @@ abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) + val id = UInt(width = log2Up(nTransactors + 1)) } trait HasL2InternalRequestState extends L2HellaCacheBundle { @@ -284,32 +284,36 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } + val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + val acquireList = trackerList.map(_.io.inner.acquire) + acquireList zip alloc_arb.io.in map { case(acq, arb) => + arb.valid := acq.ready + acq.bits := acquire.bits + acq.valid := arb.ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle releases, which might be voluntary and might have data + // Wire releases from clients val release = io.inner.release - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) + val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ + wb.io.has_release_match).indexWhere{b: Bool => b} + val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + releaseList.zipWithIndex.map { case(r, i) => + r.bits := release.bits + r.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + release.ready := Vec(releaseList.map(_.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -321,27 +325,174 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Wire probe requests to clients - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outerList = trackerList.map(_.io.outer) :+ wb.io.outer + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } +class L2WritebackReq extends L2HellaCacheBundle + with HasL2Id { + val addr = UInt(width = tlAddrBits) + val coh = new MasterMetadata + val way_en = Bits(width = nWays) +} + +class L2WritebackResp extends L2HellaCacheBundle with HasL2Id + +class L2WritebackIO extends L2HellaCacheBundle { + val req = Decoupled(new L2WritebackReq) + val resp = Valid(new L2WritebackResp).flip +} + +class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val wb = new L2WritebackIO().flip + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) + val data = new L2DataRWIO + } + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_coh = Reg{ new MasterMetadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_id = Reg{ UInt() } + + val crel_had_data = Reg(init = Bool(false)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) + + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + + io.has_release_match := !co.isVoluntary(c_rel.payload) && + co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state === s_probe) + + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, + UInt(trackerId), + xact_data(outer_data_write_cnt)), + { case TLId => outerId }) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // Never gets mgnts + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.write.valid := Bool(false) + + io.wb.req.ready := Bool(false) + io.wb.resp.valid := Bool(false) + io.wb.resp.bits.id := xact_id + + switch (state) { + is(s_idle) { + io.wb.req.ready := Bool(true) + when(io.wb.req.valid) { + xact_addr := io.wb.req.bits.addr + xact_coh := io.wb.req.bits.coh + xact_way_en := io.wb.req.bits.way_en + xact_id := io.wb.req.bits.id + val coh = io.wb.req.bits.coh + val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + when(needs_probes) { + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent + pending_probes := mask_incoherent + release_count := co.dir.count(mask_incoherent) + crel_had_data := Bool(false) + } + state := Mux(needs_probes, s_probe, s_data_read) + } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := !co.dir.none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir.pop(pending_probes, curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_coh := next_coh_on_rel + // Handle released dirty data + when(co.messageHasData(c_rel.payload)) { + crel_had_data := Bool(true) + xact_data(crel_data_cnt) := c_rel.payload.data + } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(crel_had_data, s_outer_write, s_data_read) + } + } + is(s_data_read) { + io.data.read.valid := Bool(true) + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_read_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write } + } + is(s_outer_write) { + io.outer.acquire.valid := Bool(true) + when(outer_data_write_done) { + io.wb.resp.valid := Bool(true) + state := s_idle + } + } + } +} abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { @@ -352,6 +503,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO + val wb = new L2WritebackIO } val c_acq = io.inner.acquire.bits @@ -423,6 +575,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) + io.wb.req.valid := Bool(false) when(collect_inner_data) { io.inner.release.ready := Bool(true) @@ -482,8 +635,9 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } + class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -499,32 +653,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) val crel_had_data = Reg(init = Bool(false)) - val crel_was_voluntary = Reg(init = Bool(false)) - val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) - val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) + val curr_p_id = co.dir.next(pending_probes) + val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) + val mask_self = Mux(co.requiresSelfProbe(xact), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.tile_incoherent val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = + val (cacq_data_cnt, cacq_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = + val (crel_data_cnt, crel_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = + val (cgnt_data_cnt, cgnt_data_done) = Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = + val (outer_data_write_cnt, outer_data_write_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = + val (outer_data_resp_cnt, outer_data_resp_done) = Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) @@ -537,22 +690,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (state != s_idle) && !collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_release = co.masterMetadataOnRelease( - c_rel.payload, - xact_meta.coh, - c_rel.header.src) - val next_coh_on_grant = co.masterMetadataOnGrant( - c_gnt.payload, - xact_meta.coh, + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), + val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) @@ -561,25 +706,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) - //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, - cprb_for_mvwb, - cprb_for_cacq) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) - val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, + UInt(trackerId), + xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -604,7 +742,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_grant + io.meta.write.bits.data.coh := next_coh_on_gnt + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) @@ -641,101 +785,50 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) - pending_probes := mask_self - release_count := co.dir.count(mask_self) + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := co.dir.count(mask_incoherent) crel_had_data := Bool(false) - crel_was_voluntary := Bool(false) } state := Mux(_tag_match, - Mux(_is_hit, - Mux(_needs_probes, s_probe, s_data_read_hit), - Mux(_needs_probes, s_probe, s_outer_read)), - Mux(_needs_writeback, - Mux(_needs_probes, s_probe, s_data_read_wb), - s_outer_read)) + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe } } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_read } + } is(s_probe) { + // Send probes io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { pending_probes := co.dir.pop(pending_probes, curr_p_id) } - // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_release - + xact_meta.coh := next_coh_on_rel // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(xact_tag_match) { // Hit, so merge new write with released data - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - io.inner.release.bits.payload.data) - } .otherwise { // Miss, we're voluntarily evicting this data - wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data - } + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + c_rel.payload.data) } - - // Voluntary releases don't count against the release countdown - // because we will get a further release ack from that client in - // response to our probe. We don't send a grant acknowledging - // a writeback or decrement release_count until we've received - // all the data beats. - when(co.isVoluntary(c_rel.payload)) { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) - } - } .otherwise { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - release_count := release_count - UInt(1) - } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) } } - - // If we saw a voluntary writeback, we need to send an extra grant - // to acknowledge it. - when(crel_was_voluntary) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - crel_was_voluntary := Bool(false) - } - } - - when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(xact_tag_match, - Mux(is_hit, - Mux(crel_had_data, s_data_write, s_data_read_hit), - s_outer_read), - Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) - } - } - is(s_data_read_wb) { - io.data.read.valid := Bool(true) - when(local_data_read_done) { state := s_data_resp_wb } - when(io.data.resp.valid) { - wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data - } - } - is(s_data_resp_wb) { - when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write_wb } - } - is(s_outer_write_wb) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_wb - when(outer_data_write_done) { - state := s_outer_read + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) } } is(s_outer_read) { @@ -754,20 +847,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: set pending client state in xact_meta.coh when(outer_data_resp_done) { state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + s_data_write, s_data_read) } } } - is(s_data_read_hit) { + is(s_data_read) { io.data.read.valid := Bool(true) when(io.data.resp.valid) { //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_read_done) { state := s_data_resp_hit } + when(local_data_read_done) { state := s_data_resp } } - is(s_data_resp_hit) { + is(s_data_resp) { when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index d7bbd25f..c7999f4b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -117,6 +117,7 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool def requiresSelfProbe(a: Acquire): Bool def requiresProbes(a: Acquire, m: MasterMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool @@ -253,6 +254,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -391,6 +393,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -541,7 +544,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -697,7 +704,11 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -872,7 +883,11 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From 2ef4357ca8492fd08e00a94692e95ab7bbdd82bd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 17:39:23 -0800 Subject: [PATCH 273/374] acquire allocation bugfix --- uncore/src/main/scala/cache.scala | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3cbc8967..932392e3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -293,14 +293,19 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle acquire transaction initiation val acquire = io.inner.acquire - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val acquireList = trackerList.map(_.io.inner.acquire) - acquireList zip alloc_arb.io.in map { case(acq, arb) => + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := arb.ready + acq.valid := acquire.valid && (acquire_idx === UInt(i)) } + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires @@ -367,7 +372,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } @@ -500,6 +504,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO @@ -539,6 +544,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) @@ -689,8 +695,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data + io.has_acquire_match := co.messageHasData(xact) && + (xact.addr === c_acq.payload.addr) && + collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (xact.addr === c_rel.payload.addr) && (state === s_probe) val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) From e62c71203ed2b0e2deca8d860c9e52ad26c94e5e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Dec 2014 18:50:37 -0800 Subject: [PATCH 274/374] disconnect unused outer network headers --- uncore/src/main/scala/cache.scala | 2 -- uncore/src/main/scala/uncore.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 932392e3..d871e652 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -413,7 +413,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) @@ -712,7 +711,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data io.inner.probe.valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d98bf980..e8e82d22 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -198,7 +198,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact_client_xact_id, UInt(trackerId)) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact_addr, UInt(trackerId), @@ -286,7 +285,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) From c76b4bc21ddd1052195f593bf6071bc2e7151634 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:18 -0800 Subject: [PATCH 275/374] TileLink doc --- uncore/doc/TileLink3.0Specification.pdf | Bin 0 -> 123869 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink3.0Specification.pdf b/uncore/doc/TileLink3.0Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..363f9b03d037b340981ea2f27c752bbd2741a20c GIT binary patch literal 123869 zcmb5U1ymg0(&!u9Ew~Tv4uiY9LkJLp+u%cR3GS}Jf;$9vcb7nL2<}cG_#6J;Ip00^ z-uK?Rcdc1_Pghsfu3c5#J+t~pqb4Q8!okXoOtX1-{1cgj2fz++FtI@v5@OTxa5M+7 zNg9KVK@OJ4Y-+}q<}LsZ$axK95fNl_d$Tt^4&eW4@TT{s?QCuj{+BwNhPjJ_tFtM@ z9^zNc*}+uP9P9$nXM-#bVAC>p2LssT?I3m%|C%NLHOm_!vq`&yWi`RZVDmRyS#|&i z@4rJ}lW~A70g=i;q@!0@$=Q?pv-=H}+cYU$u$ z2{LCjb+BV|bq0w#3BR4A;S#6el!1J}kmb#tSxrq@Eg@+zwPH0lbA^Z<%`99Xld`;| zhAKC=k%GC4i@A}6xhpH=1g(p)uoSI}J=jy2gVxo>QdnF@-9}PHR$Ix<(ZmGgqNU=j z;r5Z!MZ);Q7k(o(NfT{#hPSGUh(H3DgrpTxJQo1x|61*T1pjIe8EX(2k`6W*5F{um zb5jR1NZFOl?JdDp0Dg8(4u~&+3)tD**bdn}GuiOC%zVV|Wkc2J6JW))OAOKMgz!~` zU>GK75z`Sc`6zi)`fIsdVR_cN?NZ4lrbvOtx)faf7x!RFRQf?S=kQqc^>*9O?>8L_ z?Mu(b>+z7E_cMm?-=k#X4L^#-zrmufFZ@R})BMqX8CeAy`MzF;ZuII#7)947rTpnnWwWfqK8sR7m6&$0+9PKA|;;ew6`xSWO_iEFSWXVO!g4RToLmf9{IN zB?+y77HW2x3AokQID-Qa#%`_iB96XLXxK%g=7Hk;Y1@CcnIPd2JcAxbFJ|I1kM{*! z{##w)o=4^w{ZON-+CbtK$QV7%awgNzX@b<&KIeL1k&wA~(H>OJq#)Dg8zE zm590ID70xhpl+y@1CD(qVVOI8sMHRiU?acI=;VtKuc3cro0Ro z^_Vg3mqhuIm1nN9Zy-}nY9Hd``~IyBDdmGdJp=9GQRxXFAwSt2OL0XkVzuNwn@z?e zqsFv+4G=HO5|_cyfv`dKCS}GFSH2Z%SLUP4p%im|LM!jj@TN;L1SiX|8nCOAqT&=990KkGig=SkJi$e4!GcPaQv zw47poWSRlu;!g!uX#Ii#+#I7}Gwk&{)kGw8A`yy+`^HD0aIS_(;_4V!?~&pct^oG- z88w<{DE+c$#PK7TG65f(lJRrfbL4!JOQ&VbQA5F{V!8XJGaaIMtIBb$Gqb!zwZIAM zM_UHC`r0&=Mv|%)w1TH{oGvK~w9+m#DPLr;%hfI<5fo=GhE>F2PtLNRB5YwiSg}N8 zo*A27uK&od;@vYKn_t(4G-PP5i4}+q$%MJpw)0}jTRuoIJ)6SZOHsVMki>Dw;fVKx zYE2qxEQ!PB6cT|)cB1j$U8;=01*il@@B3m#RcDrfbhe*7AEDHbLeFGtZy>oiP`Wc+ zOI4GiwNFWM!!*wlU&p{~7z=7oC0$VUWX}dD)W9y66g5{<{sAnxn^5hp$B|LzW=-bi zBDiEc%3S&SN9&GIFN!7n@Y!6yC{9&U6yANLYMP-DPJ3iU)IP}28k%j2kZyACO;Ng> z?`yj9AUV*#FBqeC)x?OSsk`yF!;JPlgj!k6{4)zv^qe9=D|z`&{Z2WGnqLjKNcx+J z{@o843GQ+8MpAs&K$R_z%*pWKPjuO-J6U1g?VbU1qMiJsTGd8oJ{O!D20kA8JC>*e z+{z)d^IZfs9~n8OB6ELg!8I?W>o-2@u=R~XS?_9`G`1}LEL^n9$CV*t`m%yf@?$Ah zyLQ pO-J;f%oH7lIvKd%zVF`ps9`E2dM)y7+}7a7o6ZImrk=cR_RNbe3eX)`+06 zj{NAbtuyW)mN=ZynjF1#5~RHssNBvVyd9#ed{Dh2`An{_jZ_i=`s^sSn}CY4p!NJI z7bYv}z=~CzR)lteKn{3EFR^cr#43NaoOadxp7ukh*%HpXmK;BH$-1Qm-mANmbC|rd z3q0MwM^L4s>uF5h4PQ$z@KfGpIjtN7)Th$mrY9DpEbQ;&736hKzl^w%aC6 z%*oqa=^Z1d?1>Cnl4>XG(stVF1XUtP!681HLz`P)uhY@lHUyc&+|!Xi>iHh+#5R=0 zxO%`XMOAck1b~(mL6srd9ombZ3l#-ce$hWFgXJw(ih_GQ0}v?_l(qf#;A!`b^>>QH z{-$Zo))kNv_qr^hSH$;%E|AGE?XIR25$(#$?{<4F2;aBGPRUKhq4v!cdmnO`4mRiD7NMCE2kxcHcpOg!8&J$jHTKXgfH zb^!!neu)4C2H5dzBP#_JKPe3A6%H1fHEIVGaamGCV8hE-3`UY}efLU-p_!xvAr769 zT75dI@j=n!g2F$Ffr zbm!K3VCA<>u?m0dKiCs<%Ip9cBlL8~*YpafbnR)*OX$h#s~0EW^SDWTj4ncXF=^(V zJZg!V$Jd3yL)1;@f6?c2>Ey@w zIwj>>l=9fePGabHra#4>S%!x1;B+A0*RIPrwSN>qr-tP%mN{~#-5)13r{rjTG^13^ znM{?V#W-wE>yVbS8|Mh2LT;HAftE5+BwGf6JGc_nvArhOWp?top(y#KJ{l*nlT^m3 zpGfivf*GihcO2;~^WUo#X$sd+FVUGFmbSb{TKR}!&M^L%@Ltc_T7Xh_xO1WsvL4** zti+`wg>`A^`GEqdOe)T%bAa6F3~qw_zGwL~DJN`LSXv61{BF?}R;oRs2Fbd%mYjj7 z!1<@}v4Ut7OzAaw;)ZJp?1O~s=Wn?lwQ8nbg`A{#+N}T$h8g`J*02a7Zhie(wCiuy z?z~urlhu@BJ2C~YUa{zQwuL@dPS_)C1@CjVJwk`65_JHP!U9)XBRcdnZL;tgvVU01 zk%2h?8!8hnhTTKVDy)ocB+9APVJoHU=Bb^^#e)D$V)=MmBzP~m$y{Q>X#S!jlGb=t zK^4rA0UfLSO$z;Q%Q7Y!he8eRHdWfa)P*N;UE(-aydCj}F!g=FSHjrb^x&dg<_%vM zb$io~ozF1u3msIK+k2Y)#FCm)j+|e4F_T$FPhK1_M~$16PZ0F2Nm5t(?eR3#Fj|yO zEOBuw!kBQ{H?MLn0yR}88FzdevGEOI@N}QY;qyTEjaw1KVG1t$$$)x{EQu~5=)@N} z57yXjGqDJlWS&K^d^k`dsYe8h`1*2HjINU~?IE|N6fFx1WNb@Kg%GG`U=P720-s%&=m&O7yzZmQv#`>@Za zq1hNmifwqmM9MVm;LYFTP1f-fi+_ltsOyST+VB<&85eP3^`Mynm71B7soP{QwV>PpVg~Y5G7AN?Y9NLB zSCk!io1b77NairwCRTJWslw5CTLAC!+Yr~D&`2?EU3DP6>){PKfX;ixO_1Sq3FCLEJgytZQB2dPk_B2%a*k9TuSVTche74U0(K|{VW zHKNIc55`nV7|~_3+mF9-+r%FKAf&ROl1Vo6leEF;EM??CAdLCC9f#(bPH*VG`4Wfm z`u6iUSCgX`JFSy`yD-7eMvxQc@I8lum*`A7To@mC>_WJGs-efPV!o~NH}>={KWS|} z(WgRfRj3+OOi!1CDG~AXpPMenj)5`Na(K|dSrlj7kv#c>$(?jPyZ89AqH=fT=T0|q zJmChfW*^G_UaE#jPYsC@ri_YzjKucE2y;gMoipwXy2<6DOyoNwX26Hn0d{U0UjlnU zEQscd&2gACjj^FLmBT&@F1V+C_4A)aEeN$du8d?|F}{hks7~H6vuO*E|cm z$9P~v5X$U*#A&eUl_-6oI!rTc$dah6pZHX=tGrKHrh*T&zJTSeQU3Og z3Mt%f7PBKd&X(Rf71pEf`|rx&!*J1JDo;0SMcy&y0c^i32Uf*ti|ck4aboEm`k4WSdv;;db)XyO0Rs4z2 zlBkN2>v<0A@SM#OB%Glk&(uLes-3=~Cu6-3J-~C}MtF+Fbh#3lGIDlOOKh0kt~G=qUw0Q97z1_2Y8Z?L77o_6wJRbqqy zvwN}Nyrrz_lHQ$hEvF1HOXrk)mX3#hnt#Pa>Jh6cE}w_GTez7z)j^@0`ai-kH=NL2HthIvAc$+y$y1jQCDKa&C5w z^B1OdPV-`Dv8@c4#WSv;?SQ#aK5;M_j$IJ!*Jyi*fQrz~Togs@WZ4xN@+;caiR}sv zq*S6ZD%8@RF8Mw|u8Xm;4x)FHdS4|4yQU36Ud zPq<)>Nk**6NIDe4sO)U|9n^(F_c9p0_@!@jDUJeLEfE43s1ul>R4#-%JD1{2vgNrd zWi5~0ETu~WcwcCdYe{etz7%~v^tgBYti3)xQxF2xx|4dU_(7lpuSm zOFVOtV?IIx6XX#X88y!^Z-za#=hyw6ZAgY(p{=(i;tY<#7Ig@o(y8Z0I%?BVMa%d) z)|6_VTONF9ZY1RzsFdAXBQCeFHJJedjE1&c(N4yXd|;qB89ow+3ZHu?H7~?DXc(j( z8HhrsqH!)5M&u?=C-6=y79LsO(OPI<-6PS7@VcntGu~WHbhbmnE)k=!rtzt+s)pJf zP27B;@sC?4pQYh1Nt>DE=?WSCAQyMy9vfu_Pv#)N4^sPW=yIMRpL|tU!^AUGgj7@} z>c%a04!UtvVex{}$z%b9Fvg;QM9r9UwSB2DS!k(0P6pQ4`pjGvm^+rB+1wl3?DB(g zWM|420ijV$cxoe%@n?Ma5q$l2E+vhc2*#CwYYy~F9r^ebbie6m?SS#*@*hiKtZ0zp zA;n2bQV#E_e~tbsgPzd}$Is2}ytsgR)TnWgeMFDj5^*G_C60XRuV#>)v?8@Jm;I_1FECNiia@ME8Hn% z6>0aRyhwc-jvwo?fUf#M-WO3NRYs=qiS-!U9nBdX@aa31h<-`KqgK{Dd*P~k#yX)3 zAr;Ki%x+{++$^xuXZyVa!)x3<+ITmZ@D11Zah0Vd2g@0+ax3Cq3-8jSM<*(VJb4)% z06O{3Tui0w(W4k)OC=}2op~8~>&lMpwd`vZYyv+(^q9%Kco_xL45CMgDP{6&>n7Gp zR>$+$qV#u4877!X`=~2$hj?xKa}eA0{O_fHpCUiI!LH*3;S82tZ#b%Fq3AaGE2ky} zEX(0ypjBNr?Y=V{78+U7gS8tj1fGitN&8BLzJRt(UXqjz#(NbGnp}0hPP{MclLFI> z&(={nuLgSuSZ&{w>2B!M5GC7zT2S~&6Uz0Y`XZ`U=Kr|m)Ur5(Hk}Pt?dkpNG$TSs z){5I%^0v;UaAP7fdV8Z$u5tKV6gmReZy8U~xmOMv1rO{L%+K7Im+j-56+=(hb_@l~ z%14D9@@Hl6LsSXZ|NKx0os8CMb1@PgY}^9C{I30UO5_uUqF?=mSUTV=J{lF!8MoLl>WvyaWAa>W_1%55V5dB5=B@D^@rB1)fsbU=BT8TVx)+X2MEAYl59g z^$+FsdpGMiWqg9s#LCZgM;!^C^6spIJ%!fbbDZ5=}g*tn}6xp`Pl8(5+F+ zNO?NxZU?{SNLju+2zXHroTBYBGES@Z0)VqVKNk`?8xw58fdXBgxPNHZIN-uC;cHiS z8O6z34bYY+b4S<@t*+OB8q#v~lSf0j+I}#e$lThQG5g=+oeeSCK0cG;`ynTqo!pY* zH!u9B^^Kh?sx67275DzvRBsGqKx~ zBj$(mvD=%X6u+}WT4BtQ!7jwhigxE#Cl+LFHT+(ks@JHUG_dhYXbDZK74PuK#jZBS z?i%?F&>{aSwP7hB4z_9^j;_~@(~&$4i9CUK7ev365!xzIjRjw@uyLOGJahSlrOs87qdntc^S?t~$9#w8^@*A0(bQ@^Ns<)BI3$Ik*R z?{w|SF4=tTfM&(2)yF@Ymp=KvQgyG6&U*epUVQj@BG&oG>B18|f|QK$L z^|Oc|tF=-q4W%h+5Bc~Dz=VLWG%kSq=J$=eEtm zT~pBaEipHC^3%azN2{)Y1M`V74CtGO@EpujtgZEQd_}bDF_&@w+b@DO6-!yT{XH_&gsCjA zRWD&US9${lE(S`q#j$h zAR#CVuCR_$>>bl$sm$!*T+IN%6=U#zLl^! zXnfrzBgA53h08F(>OpjbWDlM%i?tJzb4Ynl5$?OqptS%Mq^fIYvF-zG!?o>q8|QN@ zoG9;DZ+|mru~JIJjO}fC-6jU2+-+j0vbIbkD$@eiH0xA|l}Vrv z)A5`c-Rhhyup`s@`Y@WrkJ^3lo1~bE&A@t2@yZ?CB+iHG4~2@*-xD*z5IAd6-(qycdC9c(C4cDa@&m+ z3>CMQDKnVIk_OnOS%&U>x>M_v&t=sv2O$WQYC@KE>22_fy0!9x4zj~lHTyLm3;F1_ zdl)C4t<``W`WsY0p)0|E z-JhXOha0E0(?0FCUur3AHTs-JUun}=Tw9s56t$^$c-mx1Za*V{$@!YQACD&t;jB>_ zKPr=HyZeC}mT_O*H&8J?E=J^>=gJ(!|F{&oZyz{EmV_Zf}g1(n?y1 zSH%=&WT;p-KrlqL5f&HeIO{ey0lIpqF)7rYu)PcX_&C1)b=l@Zd*2B;U4)){0hO10 z_rrF0>-~@ZwulkEt!(wV3H(pl-EGeVtA+!R@BG=_kHkA|zmme#istRE4(crgXbX?JDAO!Es-J6uHOh@G{UaQnW?^n}W#;!VTv)S%J^Zv@I$7 z_9bqQJo@Pp?ST&G@UN`E!r)z&E#yaF3vm2u%WZ#|-F9N*zXW@~o!zR00Xh(wV(9r~ zgK-L6m*7H(-au4p$-Qy5af;$L&-E?Z@N>W(dt~HgrrSWG0C5$7FFOp34_6j*Ge@jEn)x zzmKlqme`_3>SiX~2?+2LIuv9qDJ_260ctlzKlL#}Zc@WAKU;VW%mgrMSlS6qQA}k( zOseP8gqdWaGU~*+RyY09!A4~gx-^Q!4^YL$GA&j(m&p1A%~o0!c_ExiwM|<$)1-F7 zP^Tc+RH7C`{ZmY1Be)jB3^0uWO$)N&!(5Z9Yx0bX>z_Xl?!Rf~ytaAQ{ES0Zg5|FEBVS7AT)Pe%9KT8ZM&>Fdh4&&9r1R8`S|)cHc-QNK`z&CYN?fXafrmjNxj#*S6~Y(iVkI;pI2C)}C&tgZ z3wAuz!;W~ z3+OvwP-MwvkhM?nPiEZSn-nsG(OmUJl%!KGQ*T9Fz$^@nW(e^IJK8*TP5S~eKZWx5 zp8C?!4!P08)-DIJLpgts(=0BrTiIKx=kWBk(C^O0LOx1I(^JnHj}|Deq00EdNHB_5 zb`+8Zd{Qlrbw!A`xocQ6fgd+|`WTO&2VW>sy5jb50v8 zgZe?}f607M?t(tIVk5GKr*7V3&XcqJ5w!uH1oumrV(s+wgmn(#q$3W%S%P9D;Da=; zo=QtZMJ09QZt~{DGWcts{%`CRy556itSQyj2RFFcHXGGfvR1A@Q8$cqd87JA7g*14 zBNK~`xNk4GBX|CowbhM=J()eiQ)nWZCK$p@oX(}k3)Sp39>)%+{7%WRE(%tc`lX6V z8qG2`_{{?9U90LMWS+&>HhYFuFN>>(CRUpaU)K%tq5ObN7Zy|c-t``@Mx01J)d8!m zxoQX(t-%}$!_|>{ZjajtD=b6+72tq>KWInJ1AEdbH9juP8Xta=>0{CUoGK(QmQL8U z(_x&~s(SQo#r7JLVbI=n_oa61ZYAl4n{6=) za#T849F#R)p4tN?eKlT$c7E}5reb6Qt`g|fG0RIZ38bJ{W^QG*u>SkGd9RO^7F9XnTEqz+&_aa(&y=VLVL@6qHYApcMxZ+kMUCa5%y%W)=N@HoCi zB5VpY#|5%On)gYCIQQeki_}p;v|6Rbc0i5(GY&p(ASx5LRepd1w=ept9voYB;Rzs8 zqCPyd-OeuDK2|P5rP?IbFBv0DaGLy#o$U*+=p5e@@$8NjCE;B03M z`cI_m4W;^@+`n;F|A9ljF-;I(62>m({}5PgTF&MsuBKMzU}a4RuVoKGfvoKK{Ad86t6m!$kR?)I zxPZ;=Kh#QR(AS-mmMP1baiwDncKaQfb4H| zWz&7b=k$5l-vB&z2r$RN$_eD>F$8b{AyPhmRz7a7x43!u`B))P9S;Q3<6`CHJUUBwRI=3{5&=ivqLKxQ5wklPRd zp}Lgha&83VhSyfdt9V!^HvM z;f5^82jqpM7GlK%d<%nzog3m35-2A>H!D9U1U=;AVrOON<@*nH9xfon69+`{?`0u7 z=i_4q@z=B8h+CN)||Z__-h;Bnu~yla-U5 z7r?>J$H~gh#lZ#O=jLSPHiO7$idFT@jpX`r(Pc3 zTGGD`N?Y%4J36R43KGGH^7h3O&``Bl31a>s)YR0pK|fsm6^Uv()v zS}oJGEZ3}j2lY<40tSk6d3nvQPv%w{N{S2`Gw0*4A9K&44~3TRuGrpLIT$SeC{y;7 zx_fztVK*9$U(~D-8daL?G9;xitNc^W?I-A%eG1X~`ex#&1;(McJA@#nA4@k6 zt_z5AqOt5n9wSH6>1#_4?-oy7j;h?_fsXh@ zVb|jwZgyeljULLM%9FBSsCqD5YstKI{84uSP;l4dtoZe?dhtH32uynl?K~9K@SSFbEVrhgy zL_wT`Q|r>6Oc|lZpkkX4$;3sJyv4xhYVl-u_)D!I&>~}lM6#B@x2bW?1<)+w?z11k zttgxh1D+H1RkhS=fIFQKz&q$K)(<(t;XCFa>-DefHQ2@|5?tJyo}q{8exh6a%`Xo9 z%&)N~c%_)K+g^s#R8QEn6s-8R#_yjIW+`tM#4%Wc86Q|aNU_Xb8<+1H9~eKI@b8}+ zpCd{dm$Q@6B2j%NiZP^mLPMR!_>99q5Oj{U?yqIQOSs_0<-aM?=>QMfXx^XBZhlfK z%7J1vJuU3sWeP^BpL-?gWrC?SM5kZ)jG1lLNLflhHimc{j9`qn-y{>E930KFBgd9&sT5tZYQ4HB=1+B$OG_@J_{Xmnk$k9DhkxS6XtO5OkL3hVo zue*ico>$MEvP4B!#PP26ndAbM5G|=}TW)L!c;JT04gk>+r1P0x(hR#=t5NB*SR>~ zS1?x-*0pd{eq4t{H>hVv=JVwxYOj7t+bNWAqBE!quRk{2*q7W8F@4-&#!ZHc+}t-{ zTb2Vos$}TW;N?q_DZ^3n>G3DN*dL^J4XSlytB60bsy1&W);QNLeJp#3$chh|ZkQ>o zFsLHw_I@^p$_Co1m~;;LFb1W+ni~9Wd$nlY^!(h3-)s3fqaMm5&09%y96^%6G;v^v z=5MaQk;}bA+F{H+Ris{22Ed*q4#_la2&+%DmzxaZpqfJHdoh%cIN20S`VJlkw!KO1 z?D-}O!$g-M_Wp*7-;G1+C)ve7N4%B$m#VLsrqcA1A7GVd`H~tC{v={_Y;`Q|LrHsb z#@Y4<{s>$7cIien9%8(epN0*TOrV7yRdaw>765Orvek*KY9?O@?R822Alz?L1c^Ty z=eSdW@C-3SZ<$TEds+UnUWK{z4BYSaoo=~@;j;Q`yAv_i7}BnzFXUIJY_^;CPVl** ztVZ46E6fDcJ|o%Ben=mGLHJ7)MLwiQUR9^+C_NzZ( zN{&6pN&SLtj0Mdi-Au?~NbsvejWzHGuWbsdgGRQXvtzx2T-2zA-fd%LaJN7iIz1}| z6%t|JSjkX6^Y19yViNPcN*w}b)7{axK5j|8!ZiYOigAjjM@rg8UZxI_Uv0m&eNHPv z_J|K1b4TBfu8eHmx(YGby;^J)`V_#2lPcN+*W-RDw&cVzE}CQND;oS)zJH4Hg=@={ zV9NHxdi?rVpDn>=yzJzsc9Z?Wl*j(l+%%Ffai(Re*ID;=W>yE(K~raTGA_2*zA+9# zhx_zU{>WXx1O0=G8ve=nXOEm5;V(db0epA4OGt72O?(=hNUE2P{%c$1%fC-06%jav_Hj?TiWxIxlU*3rQ*R_B2g zhry3z{(5cj&?7FM#9f;=e~4zhDU-PL-Z(lcqHzEH|h zRVNf45ZHpx(^SveX)`iwKDMG$?*{%-dGe{@;fTk!y5+Xjbz#GC`ys=3zHW0bx@pyO zrD#ImzH58Z=+)x5vF+^TRQT@Ytn-oF-SJTpL-gt(CM#C>r9V+s_&Lfl^=@gcYoo5o zZNtUS{U!LBt@GxJ{PkwE908ro^Yr4_fyCpwvT5V1+lJR`AV#e3&1H?M$n(kdHr=$2 zty8gXAolTKjU!7-%&Pd)4ZtVbm(hR;gkz6OhO>@?w#Dt#`GIbo%un-aLpzbEbg(?! zruJK|MY-ubv}?e}f=|UF6=!NW^@_KMUP;0;*&uRrqc}Nip722_MH92FxcMj#B6}QkHh`QyAJ!^=qAaGTzg?0O}3q`!{h1Z zurG7iUY<^|uP#k$&waYhT{+AL->G`KeOl*Ycb0wUYhgxeI2BH}N$~-LH-6>^?qqlMXY5tz zuE3$+#(W6G;n2LjS&n#}fh)hse|$ao&Gi@WEO2P^{6cs=wy{t1f`}xj*qDO57tbnS zp$9GWH+wsrLpPQ?c4v_E1#3Inc{kTI>TAf#CX74AdMMssVsA9wJ`=0p^3M#+U(GMx zF^A$#!=Fw=8BQZH1P(*yiGs1s(tX*gjx@Au2&GBpZ_u8`-#)ho6us(W63*~vr{>IkVs-z_TOd&!Wv4}Vd- z#X}PKqVNEh)L-8#l!{3+s-q=`E1dlPYDgW=S0d4{gPb6hAdft+E=&>BEB}{Co~Ccu zF+m26J1RtzibtKRVH%M$Ac{I)PM~2{jz~JcGJOT92AorN>$THl`Pe4?gF! zy&yVy&%POSx?HB&`*bB_V>$-P0W)LX4qZP= z|B}lSLyDsRqExCYJo1N(V@ds^Ktcm4+Tsj*N&b4i$wHE%hmei1r6$`hm~uaFj4umH zW&a%=M=6{dyK~UT6E$Jk2of8%biuXY8$0P3qWPaghsY|CBkforXlU3u5~Q_50oL^>;*%SA(+B=*J%Svf{~ zt1B&KH(dOIbR%F)0TeneYJ;}J#T)eONzc~V&iw=0DV&zu zVFuQ+a&umIcJxguX&aC~=`WdFSLm0-kbGU<_mlq{x?LtKS7o%g*y|YRm7iZi|Gg=l zHMR~B$x_m>dZ3MSHohyDjCW>qj(u%c-mTuFB4z{%{zNSu?W1M$U#6Y~pxmvgz^D)++EBVO%;*Gz_L=$m;Y2m^m7tdv zIoE+@;YlJbqv*cfDq|hzG@-tjZRanYlzu38UkKg8(oYRkG_>%eac$BXen@5oU<~a* zld$KuxF_m?oxdLS8sM4qX$O!*wVIKE5aojlL2{p~@lpntZBM>V`B7+q)Myz-@=?Ey z1(CvV!u`6Jo`GIf`fhAf(`FUtpQ^+N2ZuU&d+n@I6#2XNZ%FUB9ud*|1a{||*e&k% zNWGaN&+iJZ{HrlU2oG^}qXV~t;E7Xxkv?@1pQ=cGL4rhLT-p1m=!%7S`~zs$Z7q!N zg8UqgSIqe^9h&UdpjEzsT$QB{T{XE2UG}+>Jp8~jMz)Pb9fPS9*IZ<;=c%>#$LY?t z(L-7r5`(q(q?#E(0gGm2lNL=KG}!V`yh$(|+$f3wJ$W}Or$=faD6Ml$9yp$MylmO) zY}==lijPs*`!K@OR@Ke1AcD)qd0q(9QBhcnmxQf^@=0iHQSC&8u3JEa@d|z2{)DFc zF-eX5>)K5yFqe^c5Fa^*tcpi-DnYv*62@(UTrn?&@qLfdBWF+{q9SIM z#D}Qqi0Odos2_+u(bGZGVLvc)p>oM`(T$-bSbDvKJP=*5Y%mQFT`(=E#wF6i*)F^edZD3bt6;`B1~5(hLb>$}K#9P8$&yY8u++{1PB}DcYJsDCUn%=z9c1bKTEDAzSTr1Vl$!BWe|H@RE}4 z^QZ&v#aw3BIvhBsS%YdxO$oVj4sen>=gFwj?qh_t?qdX;Ud5bdS1FukH!18mXIZ^! zX-#Pqfj^B37WMbdXqzD$RjQs;O4CS6SBqk=>l2Ru8fzq15^bij4BJYR{shfwHEuf{CB!IWOa2W0ybA z^HW+V*b6H9isirNs;%{@>a`OZC#X$bp7$Cjph1*>_gCi{!+VROlqZ$7GmdI{QmQ_w$8S4Gb!p4FPxigiBn~27U@UX%G=;D#z0mFzyut?%k zn3(j?;*M2t-egS-x2hEp&w-8uFTQCeepBBUH|CjcMUQbF1wRhFH0CXCB;Y<~5DoZB zX`DEOMLfHT*N_)QJoDr&Vth4m3&N)(ia2dG(PmJY#u7qucsN+OaGHO8;w(H*Nrg|lnc~lt&fyypFjOKdp?f8g=y%w#^zVxXMnZ4IN3(wlT z0!5cI+?i28Ikn||c2;hQUe;QCtU1SAjc_aR`tfRl z@!Rm!Y2Ch;cn)p22G>L4M)*T0#!)e$8S1(%87+A?j!oa=5II!b3-MEQyJS`0iUjOB z-N#|X=IFgRBI7)2iVotGO?{MM;vKY0`t_}pO=7}RI{sm zxKcFYQ1w<&h&!7$DD?QW&YYvgdEu3DRn@51rkk37{rfKS1?ZELyW`jSdi|Vp{>o;Q z!fi~d2dKct_kYJpEtp^66}%CCH;ReYvIuw80h{se@cG$?x}yqD-9jd4$zG*4)Y87AsR(7H1!fHt{q0$e3nr$lEOGYeMJOx&`;W;B?AQir>5poUU8!NrVm(M)0hD1Y`R$F=CZTX`Qyu_ zqVN#LngYxpq^5{SGOCKhGH|lyXr>0nJWi_NcrGYa_F*5d^yfZ0t z?tazMWymRlki#B9cj0*y@Xoz zpfF$ofioH^qPnh*pt{6 z1g!svJ^^@*h+k3gD|3sf5F@I0!xM{r{&`0*P3ZaTAei3*tc0iQ3vo0q(-!10u*Fr& z@hKAd8PEMxZn`{NR?cIIfYawHsYG5-_-8Y}>Q+q6IGxa2Oyvu;M%{cuRx)vAP5Ecw zS7AjGqYtlpV@AG@y%}Sv!l#deZoN&RbBPBIg~T~&TPvr6Zj*P*q;8_pXH;Br43~Ku zoBYh{!93rf@fOdN8iTR$D|&qgntSA;2UxBc(>Zx=hpDw`YrmEvWE~<{QftOnFeS)y z%TquT&n6{)diAixBc{LtYDQWo%co)gB9$>t>9ITDM0A;ul8Z--=Els*O;W3D(V~tR zULBoFO^)v7op|{>X#2R)hhTeq8NIN4y2n(z^eLR5QEG zdb-xz7|xv)$vVf|*hI0UK70}(C6ony*vwBo=^Og{mMfe|ZrIHXXFEGYox zqu9@YHp>{w(E|p8^dLKwkXn{pirDkLFDHMx4(Dte1T(4l`SQY6rbX$@iNjVnhVbO~C=JwhhT*7_-EXdbpo!;H z&Q0U?d={}p){jm3F!XVh*@Zby;H1m*wka9GQFO*nuy2Ywt_=yUY zthSM^$=dbbT;!3`CBMn@#(pN0)40*9MTQ7TtH2?A4Qg!h08)`>TN?=sjfaVqplvWGZW9}VzVAIVwh31*5zz_ zmoDsAO`Q60ktA){HzufSXL$O!VClzrLYmT6RddpMwaUQuN5~tz{L{%)V1IWaCoa|K z>c}uQQ^HVhdeSYAe1v^HJt2UC;r+b|<+^-?20hehRx&N8sb-57bBC3fP;i>dZ5C%% z8QRD3Y}42RRkEw-_tmKdF8egrclI|0jIRs5@->(y6yen8fz+?Olb5R=lzCM2);ew4 z3YyV{GRR)h2+`z3s>IUVx&`W^4atjpi{I?l8p&3G86d^7X>Cv#b^Q2qLbQysN5R(# z>~xdr@kB&9p+vcMRRuC8ObVo|SDO7Bb~Cj8Ek+@nQWguV1{I`gohYDb83%^$7s7MS zGwxd=48mBVDFU?5^+^L$(Zv$pe^d%}b6JDCO?GX9k!eOr;IIiZcql0)G^^<0WqEkC z^i6KG9O9SMLu4#Evvly~1P>;PRMi!W?q>cLsoMUGfyHJ&maNaKJ{PMz#1-m*W^+W4 zQ=H{{KXah_eMUJ(ir*adn<(QfC^|+-2@hJupZ{An=yj#xQEs&uPB;7Vmu|F&(`4%T zMVHN1Hl{#Vn4{`S#WXZBG3b-+C+xE0=+qP}n)7G@7ZQEUQNOs^eI0ea|^ZSQ9}SZ=0*;KKxHHdoaDPP3gsa?lzM+U^E0WLI`iXUIr{{7dEuXLqQ&IB?Ei?x$r%W zA?J`NltYF;QG51Ac%;ih9yHuE9<<=KxHmL*R)WaTc77y<|7$1*3vZx~^(piZuUps1Be zayHYZGhO}7F=uq`(X>_Hb`>wBEzDkAW4d4GP)SYP-B%*FPdSZR?cPWp+PsU}=b z^g|mbXWl(k534WJO(*Oq08C&>6C@|p)LD?cOzRats@|I4GJP~YG!~{fkq!HI!T0G* zONfiUdrFnV3Y^F+0YYRL8_A6hk|Wu98r=+k%iaT(HO*5Rki4! zP!WqX0X<+ta-CFIKPYk|Q$&t|PM#leI@>pI?sPY_x`%gk9iZe9L#ou`)y$+urTon> zW~S1GIgJ12eFvyNcGKq*29FcH&)mqGg84cEGrE`@^Ionay z+MvQ^F`#orV5VCns7__@t;$X`cgqmzfy(I2C!_!^zzfoM(k>dlIqN67l;i~Ce!Vkz zEP4W^waM$zFB8VIo{$623<51JRwKRFsxbT4^&oA%_fxpVm+EFax8K=v^jTs>hwIg? z8RDhA_8SKjD-dk@T@OPdGg`XK%ZiMN{rKtQv3^)kFqw|J=)@uid7v$TKs5rpxO=4# zoxB=KRz!xomo8`cZPkXPvfkXP0aDw-c_i>GMHJw?JuO5o4+m|qV$fpH-w+-%qyY?( z6EQAZihD!D2j=9wJZ#nz(8s&6kDK0?+v4VPl3)w>7fa0$bd`u~{0@5+9&UWk2QXoM zazj5rI4*G&v3DaaBQ()Sgrv0Wu#4+=P|JwU>SGhB8<>tk=X9%b+BLE=tb5z%k!TRV=qB2^5?h_y zoXQN8oRz~wD3*wwD^N1EvWj^OGwOwxDr; zuwdp3m?}ubLrz)a^9o8nf!gpo7GuHv5Uk%olc)MA&N|H?=(vB`z10TWcD>wZGfB4pE<2wX&ysD*C%8B0ctv_q{02CSmz^f3 zk@I@z=j!!{)S$ui?Z7zO1j~Ob~fXHoMwZDnCxy7@>NF!*?tsai;9U zeNdMBWAKCPK;C1oou&iCiy>&VLYL(3tOm4UIuO zSeCt8OJ;y8F;rEHAa@o5UK+Lb69_t-Gl_xZTX`@eh*#UX%m{N-6f<*Vl+z{YY>|2- zvj%g+kq$7|%d(w}Xe zW4vj`Z;_Hq%pO4M07`N9@(NOyo~yFAqpNP%;o*C-ir0SklDT$MUnayRo}oNeGESVD z);j12u^pDE8I7OL#Nu8mU_*lGTuO=Fjcy7MgPN}kJt_T`mCt-?v`@K=%pgdpna;=C zbAf9KDr(NppYp2FKabIFmzx36@s{5v-tY&ZwOz%%(nEFb<09mFaqvQ)+i{ts(*g&dQgJvOxk7#gr3AzMfK_MNVk zZxJ{aSozN@eY84rta_9Np51`Lyk*b6K`A(Kl6_RJ#cy+Xyp~UTpWJbc{rb6wO zuabNUzoWD1M@XX4%fPD-^Q*RSQZW4f?CvmN7OPI3n3O&}mAZhnB^_C|L2RzJjgvln zDUCl2Lo{>*e+iu1fF3t^HWy0wS`^5RYRjBogcdzqTRQl_A<$iB#IF0oA&<&j2@$<&Lh%{^;iGccjIsaG!^)bDAvUA zk6mTH&5&LH`(7es_gS{hVVsLRVL#HePUB9PPQT4p6I9^MnF4>pjE&u-F8fXnkVm^d z?&Wp4*<%i5Rn7d@l$hkB#^ciJMgK^9+=!aYYa3~d|7el;a@;mUbK#(5?v3>HH#O?(%)(l!qijc)n7Kc+v{!BRO%25aH8 zLDd5&AEX7g$Q9cCKNH&IU*4{Kr+F2VCv@r^4m;;29QoSrs;wg~3okPWd}_=gNnV=z z%o1p8_;WDc`h(|VwiONmROA^{^DaZA_^_W0nW5GIwDd^ypHq6D>}^4UUp>!%3g$RC zHZ25ZrmceV<6T3y^1Ml@F?DICuiua;MTK_6XGu`VTbs@|H=Rh^^fZzl)}r6L&}MTy zrjeV|reG)&@jmv>RFG|;RN!ObOUh!D2EoICjNwQrVj#oD`e;c{%FB{93q>Zx%>2=t7>kJ;17%KvI0z39gtUS!NWfLI?S|3*Z8JTZ{-*}(C>o+Rv(mJm zjJsh5dJ1DXI!hbXiljDwY?yVG#)4W1Sm{%(;FNGPt%Bau%$%kqC2&D6y;!sq4l8|ozh*h{0sx}; zla{B19}=CLKzOXB{Gnak(X(|vRev%agFKMIr_~)kwB%ybnjtoB<4%8OW#i~xj>cTRKCUZ`y+Yr~%fPX^z7e2YnpRL>56(qxCO}3L=B5D+ z_vWvoA6*qvn;sm1gKqpP9UQgGoQrbJS!9&3V6*9CM>I@L3 zW-*RA`i7>IhZIUQ5$y98BUlb1juUUr=|V`eqZ8tj#P(tHCtIyBxtkIYB zh4yyW%H-rQE689*^P7$MX!G0mSuQ?v<4vYLmtAI-Nv1cpD@_$XFU1rR4}@89P$8J= zx?ihd_jOI&9u+~#cGcaY{A~}!bXAd6;JBLO&p6MKcAa|M1cqw*;g__6MbEoVe@#Q`=$TO{jCizAEXTozm0 z&>C%Nx79oE7t(JOTm9elJE$e-ciPVS++Kg&jX~S-;ZLbh+*G^~xG|Ka+)nGy#A2@* zODy6EMuWNi;aG{&3btpx!8hA45~OaJ2SvBoX@Jw~2#}rvQ{Q!mI<((DZF0^I%{|n= zNF)hNcY@%hDjY`RNX&SZ9wJ{~gHictkE|KnknMX;zU45gs5}6S&~9R%SwDgKOYDS5Gv@izZcm%UME#o|GBongQ7z~c3>+;ZPfj7TaWQ2A(uqgVNwH zIDyWM@E#@tm*;}-u2&Y8@G-2g8tukY?aDzFO}7@8dwi*o>U#Op`sqo}Lex3ODOIcJ zd$N6Lnl7$JlNu6(_UxZAJ)HE9{faub((x3CV55$qZgyWjuLOZB5!-_$WQF1hYrk`E z%t_zr`!yxvZof>O4h_=aE;B(3PID~lb%PDE6X3R^q3HvII%>K9f3op^@#_B=(*F%s zeo@vhdHP?m8+8>&t$G;{+fS(o8X%o7Wg@G>k%8uK>|9Qz0U8n*$G7i8|{Om zq}$H24(NB=ErMCLg3GrH{z$sx%9d0QjVx6`k0ffN36p*F!zk%`eXOU&Wwo4f$nB04 zupm&%L(T`|LuNtF1Dh67?_7}wf1S2@-)ms3PBQe%3+wb&+GaE2wW_&!#Sj@ux&MD0?Z0IC{}8+X zcC!EFcE1SpS2m^pj_Lm)?EhbhHaqLTSv3o0=-mBz@y^w(gE ze}V2VMbGfX=-Ix^IRgVT=l>_P1a*nT7^riCuJ1=KvWToTyg8u)TGXG-z zbgZll9E7yYOkalniH&mR8m%vo62|B~+O(* zk)7@Tx*?u*c={;MKd`qyILICBk&e%?*(MZA=9$`zDwMLof+cy{#Y?4qX`yfCR2W!a zlmSxiM}Gc~U?vSjFeD1pZs1>n0$}|Wkm^WDjl&HOl97MzA79rGJvR^Eci-cBA1iqg ztZRNhtQ-~sLjqyuf?%>_aCq+Z5iH8TfajLs2HPF<615*I)N_ppO3smJ2`AFg}nWFQZ`Enw<%dfg4>HGXv?*Pelikc)|* z(Ka_@_AcQAa9v+{o09OP7J%+vp>o#)R>@s%_s-E_PkiFv=MAtQ@H;qf^#kS z#kOLHIdABhkN&5vSQh}{qOvs;YLPud9`E-|!iq?x1lo?)?GX1P5JWZ)6o~!gG)`Sb zP9+a41u0MePn8ce2O_)W%od#`(62a7hVV|1N0?d`$XizitkKTE>j#OZs8)?m{+)06 zh>Sq-2@dS-81LltHe^BQ<}RWc;yj!;DuNB%~QSp?ibs^cqAoN;H9n1Bp#!?8DE;q{rB! zI>2b3e=RL?q$t_kn)TV76=2LvgjZ{jLs>%f;ojhnkzMv@_xcnvpuFag%pheU--+3n zqaR=xcF6pk>~nyQ5f-x&d_sad;-buTTM7VBP52x5Y=l1=08(%FIk%rsj9=XQ2AA%o z407xVK>&nq*ku=yb-Gb$i(BN$5NHyEL`p~<$9$NRZNP$lv?+3+C<4Q%K3p-)0kJ>u zF)27EI%mv0l=_g?Rb?XPuS*Qc+JLGUFZDdBCXBz`v~lhbvlQ_Q8goLkcP2uuAExq#Y26^H7fJ!n7)(?r;V(;S3`Aj*@1kE%IU9hUy^yKIIr=b9@Qq@y5`+^NO0rf;3VLW|xWlT6Zy$vT` z1FmMO`Lxrq`g#z%$X8KpA)SzE{dG2#_S#3k;?lgzM;y;H@N!8p%Z4kX^+La%S9B3OgTm6&CEpAD8F-;rJ{+y(KgP33>m zcZJfSOyQCm7wK|L{vMhPx`MrNLFSiWei9^>v^M6K5a5ejn1PtT>YBC|bnCzUhStn< z>CgWD`Ry(afNp!Rvcz8$REIfnsF;V6P9Vt~T$63&g_aKzUo}oN3fPtSUCd9(bYU9~ z#RF?w$N<=BbMVErI8?-7=;*p*Ju`vXGW26`OH~m7>C}ipD?FacHgT7*;zv?e{QtcSj?#P z*^KG)8<3s=SB)NDP5i9m;!w%8R!e(wxi`$pqZe<#BaaThzac?LGzy#25~rd0$*8_` zO{u=5z91*Mn+E}`)RC(g2>UK(m*f%OilNSBKwat@=R-m}XJhEHk@cPqtbatD5&d@i zWoLUya$kPLJHYT=4_h8W_?_?je)m%_uXfc+hW?hLD*;|^j68+?k7FgcYVH*Q*LG?Y zuJE@$$JpKdp<4<=z6p}FaY5fcmp+4T6jxfHH6wjQZ({90A9U|&kksLr{4T_q=yoecupb2?0-1*tLNpW>ArZr@V@@BkEuJWhYCQ4WFF zeYJ8P$X@Ako$%J(DS9D2i_jo&(?*D7r1L7J)8OvptwF0K3@hI|(O@4aJ%fyT(3jGW zzSvr!69iDjFg45X21#oKt`or4wb2;0dj5>)l_~ibi_%-@b2Tj%83*~}z^We*{Z>m_ znUm`li%69qo&fYUC9oTZOs+>c9y{Yh$aljU0g0jT_HXC4R}{$tWRn3#p13rB0AR=~ zl4tJt#k|2UtFW$*hLxAHOSXSH2?ksgnHUM#-6y@|Y@SI$rGepn^=w!hL_7h8+Vx-L1CI$|5 zi(Sb0_Xe86pjNE)fJ__lrriK*|8?Y*>TZ9X9=`cI=BJs@36E%-hWxK>dEjhKqd~-O zj_8MC7z0`(GF}ioP50)oN(59-srf_ zh$Y0>g3ASv7-BQSRePd!(g`pS04M+zbv9d+M%YIq$DK?>-#^8VtWc$ys}t<|x1kV? ztUtf^<0onBx{}|(_b5P29qWGZ;DK8fZ@-utdT}*+%HqBb?`s>JeX?YKLYeThD0^ax zKaFrM)Ica5`u{LiSIEhY=-B?8e)^FAamtOSSbXzn?m%iXBm*x9J_9TPQCsHF1T=tS z+z!aj`6l1>ec_L>)yw{UR7pYQRy?v@Gu!*h)9=$4{{CZfk8P>B(QZ>u4^({QEl~}8 zM_Zj&U>;J!t{~6LH}A}@JAMO1lLm@;#(?j}i+%gq&E~`wP*-NJm5Hj>1#uG8s%8|y}!)WjQ&0i+J(7|MGA7LdBRL5I*$VQ z@^;W{LU}tTjXlPUzV0{Il9kj@-KNOB72&-0>)k%uetbfea(g3P5^cP9t*6v7^@vgS z$l(g}7ec-d1s&zAfiMG~OeN2;iMEHkl@iR*Gr+lDS_lV2^{UC>gxq2M`0Cr?!J0)F zKq}xtQzL&HF_BqKFRhRTEVAo$9r*r`rc_K}>pNX+55$bx_9w#sbEPh^Dbxx(`iXMN z8|>oF(`5%aP(tKp@FxCqGlsJJ%BT>&Fd9)35@$jGJK{ICukhO-z>oJ6!s}eK!0-K&!mFP-$l4+k!NcqY{JnE1$vo%t;-|8_BX5Cv z;A8B3p1dWhAK=O~6|8nqd7B&NQ}t=0$6vG+`dCL(X>e9Q^R~ZP zU|mVYLnQ=Am*A(Ngg%JQq7qL5TC89mE008RqUfGteWb_`BXU8}9q$Y9dnD38Y7E#? zP-VindBxJ;+K~56w-2QMPyX(l?a5o&H&?&ZPeR+ojpPq1-MDC{^0TDXA>Td68xPNf zw_F}=J@T4l{bcqbf?iAFu)j)Mx99Ym0SdU(@ASd99_a@k`Ux8w$zQxGSqm zFH$UQE7{Odf_e~^jBnUWcwy`|$h0GZ_6`5EZkLFa7e=wN!4G@7{@mHCRyxp=^IC>A=9 zP4xY6$Wu@Ft1#B8c<1f(7}hLn@VPZ@WzB}#;>L%CR=1%R=G`46}gMq5VlsCe%QzK=bYqZhG^W}(^3#{Js&Pw z%WZ_zEh1W|%d2JkY@L=-3}iPS?mgJ!Wa)SqjYkv>rr`TuJn{V4rH^c-&>5t_6Cd?y zU%tM&Zr@ECY7pW?^#9#Nk z8!uQIopz7uba3gN-YLL~Y2;7$WZ5nKwD20K{v}PNPi>!H@>qNzI?kM4!YXf>Jx8-h zGwYM9S)*B`8SuJ@;KU6jUt-fdr8{$T@`!rZ#!(%aV?Oc1fdl+G9(FGvef!7>gEW~h zJ{R=3y)O}_n>K1Mx`saJf4|-F%_B#oa?qbW#@Z^>BWQGc?6a}(qigKgP4!pz?G@sf zQB#l76TDfkP%6aapV3-u-C!w`QyCA~iQX;)U>;y+Lw4LC=PR0akggtj2jJ`=*DJnu zm{UVc*sy*EkcS=#4+Op+vD?B=kPqFD4mds_uiHTF&;?uO9^d%FvanJ_SYf=&4ZJ7D zN3ir2oR4&SpZFVTP`mgB1wmZLp70IQ%2=%A50eB!sQC9ipuYz4@R8&cgvE(s$A$iD zY>X2kd^Z#atoRlsZW#R*wNp$c#b-~)3C0a|ivV51<}ksYyi>$p9JZ45iNr6K9$#3> zykATsWg@2Kd0OQKUNfoKrSa*dX z=EM(``Ns*zCF$Vec+w?dri+WPPq@Tvt8oe9)^Ru}PF>}u91H?bRt8eit<(b-tR)27Dx4J$+{7FnN zg!|U4GsJ$sW}B}e+J4a23&KCBYuNop_l>qQqig)}%KfSBjrW7^^T*{r!Au0F{sQmTp8}|nD(bE!xJw>@@ijtoIhd z75bQ<_FUqToluHGr$5+k#0OwSso|!l89-n5rxr;8HW0gERyiT93^#Heor}EIDg7?tFWr4X~MS z&6Qp4!C65IS#>{JDrb7wuv{fP64q0@O(M3z zW_6cZw`)h>KNuX5NG{*vd;(~@=setbz3|`aK1O5yV7$%y8d5pt+w8RQ1_pyc>$f*O zh_7FN7hyV@<8yLso~>KwH19lFpAab`|HLl!P)y&fiJ-&KTwX(%km)vg75YIHi>wwuj^Z%>*jUqrN_E^`vte& zBVNsG%%bn$Y3J@&GXE`J@A+z2)XKWTQlqTksT%Fgty2cO^-7^*LFQQ~T)j~Dm_X>} zp{)FEVKHf&u3Q-Ju$d~2S>kl=q9tp69~RkEDaTaH8U|F7xCf=fZgjLL4+APns9wAR z{zdB?{^w;wr(9WSwIbeNOthu&ty)ay0 zQoivGR?ThLdX=HFm=CXYfi+9Kt%>wPBfkCd6SHFK=qc_!2W;4Pc%XRVCpheZ(O#h7 zaF~(JASmp}>fHI@r4`NNV$fPB@ZS@Vk5q6a;a!kP!fOq#r;y_HI;11Hi(cM~qH#ne z2DbwYksw{dHJs*%rxef0iQRu%%-Q)E_t_X^+Zz|?Cf!z`VX9>t<04@)x;RJ%!_lJo zYB<+!${{Uwl&txS);VeqCUB}B6xWX!J!0mBtEo*~i>f`)wJM^;P@&F5zUUY9SLf)Jo^1Mx?6x_>qA26BqeZe)VU#K>n=8)sB%)RKg12dwi-; zQtDYXq97_-QeUni-HJsm0+$MNyRiWXo91}4k%jpVs~@u10Tk;7u-c|=B)|PErUoUU zKP)5q-3;k^?wr;j@{`g51zMz!eJzZH3Ti7;D{6IUL4mWK6}?xUs#_}_shQZAxHV=! zNeU;P?nJzh@FhH{E6hk4d6j-)8UZ^2vgvuRV#Go$&&2Q))S!XE{ zfX?1o`K03=1}EA*XG%p1ghB#)MG2p{2u|zUWIjh?qE zospYw_H7Ss-_a-SeL3loB1v6o49i%M^2=n#Me}fdT57|8VUqBKf10S4imO-yD3h$9 zuU%s(_t^(7i@dr%`72fMxaqCR_G+vm48>OVbnxuGRo}|Dv8b`&Q^PD4{er$z->dKE zmG*PDQwb{U4T?)o?Xd{1&jZBK=8oN(r6&)}M$xT>)K8-c$61G_%~S zXwKr2l#~V9IgzxVBwS&P)(gT%rqB|7QQSsNfu6DmBj;-K+CC?7_*^tpFCOc z;*f^D&Qi-F*Y+4QE63g+^*aK?I-5Y=ZiYy%0lUA9HdULbbmLcBMF)!RBPb*^%C+1P zek~%1p2jdPqM!mfg;LL0BNg+nHY1AN9~VF*N6B8sZJ1?Dn$D}ImOryWaXgME?`(L& zve!)-52`Kvnculu}KnA7i&^buKvLXvn90qz9Q+>~V>)kC;ER=JM{?|2=E zlpm)bRbP;N)r$yb+*MwHXIu)skK0s_^(^fF`2695RsyqB9!ojW&S0|L(D0^Sk2t&D z>7dGooJL{%RtHHmo5fOK6}%W4-45^l`v+OY7DQ4f~>W; z->a5ca+B($W8dP2(4`RFdWcRdOp6PKgI7xPmOS0SgNmvfo735_gIuqTeZQMhU69Hh z5!z&YWPIU{pb}${3Ti^6wxlS0*;QPmixLt!T211@P$V?+hL0+M@QfVITe9Cvi7`Ya zSVA;T#Uufmz!I&x_@g}IMn$PILH0aEkEkP0w;F8lNId36vGTq2f&03X07=={kY;Z* z979eHgT+|cO_67#Q1&ADfqkZ_ZfCU_*1i|&+aQ=1PXCdu&7Q5zpTNhzPCs2ArwA;O z@i8p!6Y;%Wx3{tgk`bg4*AA|bT9`AuprMdaIL=%Et1N75N%(Zr^~j%5`S)#>{(?P9 z_Igo#BS{M0S>s~>8b?3XrrEsKHK6g{eU_jrLts%)9Aic_E)=GYLl8stJ?)^1e6(Ds z#(R}}2qkVp@k9bRSOU?~IyHYl`-eES1)s-}R$+^3Hn7IH5}MWCy4+a)g4$(Hw(egbY-{Ta;SuJzTZb|^46Cx9ahs=sn;e+ zkAi}YUIbSz*D<%HSR}hxO>*5DOFQQ%*n1sFtEYQd2XOM;Y)xKKJWXCOyG2^zT-qJu z4v?JJ25j(+dE)0Ran)=~U8<~Dcy@)F>6^}#PT|8TP;!R1?=Bm zRch4MEy-4b)-^F%u!rFgf7~>YM10JD0IByvqq$#KYAmtl7dbGCBxtCw{I-XfVXe33 zaKSr>(mBH^CHGnrC=!4`u5%5ke&-U=I&`U8_zFp&weJ#b;kB+*A0HP|l!~3_Tmg0A zH&05QU{(aJ?uvtGl=YiXbqc%z{a13}DreP@u8Fld?a|t?z1@C<_k=#dqv|$bXr|g% znz*JgNzCbM^o!aG>rLvFY18)evyWqmt;a7hC9V4C-y~hAk4VRrx+OC+)$=AlwBUXA z^flng4)m=!ZMmLAIIk>cclWb@rob5a(xgy9CeI!dmL|iTxL9$@iL^v3GevRRj*Y2| zHv2G$(lYn&x7Ckr5GF%`&ofOxFI6NjpH?E85}(fz<0GI;K9-6~Kzn@6C*-4JU8%~ejb~fgk|E^m)Ra)b?@LR;8Ws9P{xjBu&v;g zVIJJt;_9Vy5C)Hg!4c3NO~uF0{27vq-wu%LjJAdNlzlLWUuT=LB!Q*7P_K7%@GvPU zZ`^L|56H`sVs|}G+B_-m#P3{F_p9l$tW2?>(Z8E7n%^{OLSEJt(XaZ~N4H0Ju2UfC zT8|9p_f#J0x?GZ7J~;1)wP$%3AV@iFNe*qJ(eE*+Y_;XdC2id|5LUbfqb1)m4U}^GVMm+ zm!Sr8(l)Dj4Qh%tVZF)vtH*0C8XK-SIf_bjWv&DZw*Sa(042k~2q%7MX+My-N}=%B zB_i;%^FtX0P%sq;{qjPNEJxwq^GgX`@^U3MpXw;(s=TS7jkGb0s)(bTiZo27%w8Ce zt*@8xKs%PZgj@)UZ!InNT1&%JikF+tzZO3IyCpX%iBe$-VOrQ9IK)?cN3p4+)FrHV zddhbS^lCXE1Gq{GV*}o^;PpM8a0>5L17^tB5)>Y|$KG8A2=_X{)EkDoZt0%8K}jbY zW|k1u=Gdx0Ukx`x9>~>h$+bEH1FkWn9K$M2gFsjdgbo7`8p3rWWkZI{E#^ElmmqEf z;ZIS~ zyP2=h8MkG8O`xA4I7K>(@Qx8*P~Ss&C-e_;tU`Ax?L)3>ffjl$`^2;oH}=q((%0k< z;>3rX>SJ*U%`LoFVq*#pmn_&KI=Z>H=2O0U8j6-@c%Eedq^%S$l6Q|9b~bExVxFl1 z1^OV9J7bOAejGm+p`Jn|`UJ*2B1$^nhkZSGCD7vz=gg-v>QX@WaBe^^1iIp}BXOvc zEXzGcBGrG$iG|%bzObyvE&{)xyeGD3fR7r|Y7kMIqBy8HFB!Er*4B7JZ%!Hy9qL$? z0fe+js*>EI9xc=Tw~{{;&u?9qT?Y#gvv72=t0L`UeBt1Agx6@UAXUQ)2qG6XbS{f1 z9dWjpjYKwz;%sJOT!bgS{*s1Co!Fgjh;B$d1@T@hOdy<&VVvLxZSwJ5$7C&FRstUS zPSv;)b>FTxGjdUhEc!I(_ z;4gc^uFBptKX^a&uxWfLvy$(>dO;g?Xg^`HQr*(^(l^q-BFT=cjUTQ!U8ABk{;^i? zJRzi5hE*GD{3Db;n0VcH!#Hw_h}qq${7a=ZbgT#=Zk$EARohN_H38oNKlUMZ*8LdT zE56ZmT~d7pV<56i1&v^NqF>(+y*QUlU+&+iFePo&C%a-GP06#fPnub0{qc^URs z7tKfZ7b_rN-X>BS$o@4s-OGUX;LM9_e(G@ddR$S-%};+%RG&B}S&tqH(;;ehE>t}VqM(@UEqV^(? z=}X)-oIAHuGH=yC!L^p5J~muQq3iJSjb9bq3H9H(|Lzj%Hgl>l-1VbV0X2hwJA7sx zi1y40GxXz_Dsz8wsW;jNV^7q3D!OjMhIr!f+g(ft>Mb(w&zN<1UPFlX+<1dha*FBh z^G}>7s6pljXrG{pDIi86-nh>}gYoL_g)YKvPVomIixnUXX2eq<7rtM)mPhoMNb~;P z;fKbv_&~QvKrl-C@I1tZX@!r#xIu^npd8e@4x2D>D_hV!zy8#5@JQiW=IpenfnTwL zT%~}TzN5EO76I+BFo&>M34)D`VETD{9vz5D5m+Z=QHKR-{K;AAH-7?M5}`k;7h%1%-X zDzp3Bq;XGF;9hY2qJ)~%(j}Cuf*C6OKy;B7%mXQX|qy~ zHGwFn#xgceyo~uwt7@+Ie?{hKSdvF_6C9BU*Sh`U(Mc56S5B0{nHo2akAmE0Yl*JH zgkgIk-j{^_h_{pCCsy#k)aZQQqf6OO+KT;+L01IwxW0nM6{C<$)fkBU`~yP0RE6HK zC^LR)O%<4&&9C5y70Vcd$Hj zzDTP?b#x)~%9-`>O$o=+dYL?GwWQxche1*o{yLH*I-=@@9I9gddM6ZOD$uMU8Qr*4 zZ@DQ)B#bGB9F;i?nn_bLwRuu&L!TA3#Rg4{Xm&51b-h-j&2Lxznt&EC+T~8)#DkDl z3WN@@S6lh=V+}5R^=3s5PvWd03-=kkHdr9pcyglv6F$cP?vgO4a+zoNbT|Ihb}X45 z$;K=0mD$ijuTk_mx7BMl2=PfmOgCl5rJg55b|C$TXX+NNktkEsp^}DcLEFtH`BCnsFe*tir7ap{Dj9J=NHO+vd~Gqc;c@3U3Cd~^Yo z(~2!|4MN1uxSqSc_7P#4T!jc zvmfn$Oau>!MsqyLBU z?UCD=fp^I5tXVKwn??_W8&=uOZ3#WIh;frE4EU^B3?jF>NX`_G%2XGl^m0vM`$Iq&)~$M?q%0W4f5hWT@{KGeULj&tI;|n`K4AF83UB zI*Q^LO+3uZ4CEZi<~`u`K9F#X zM4nwL~Z8R+AaAdldXa-a1F`8FMZRXkX&iVa*3jKGr!VW zqJ~HQskOn`n@C~XHJ5f4?v@`kG-K;dn;PcSfLe3`u~&mOLx^J*?CrS0$w26!Z4h}u zl*L1CTF#=#-p`<%ZW-5*$#pO>BIjOw>>}BCclOmi*qxXhqjOVSqAhp|Kl~m@WGqzx za3MfqQcs?Bt1YK02kTgRKMQFn2SckxW1&?!frd%ff^p~(L4!~NH}zEWub!m>=&X&i z2)wmaeO)2(1khkpD+i5M;=9{LEP#Vvs;21l-xdb_+P4L<@i=CH&KB!Ke;yoOsew)G z6OK~ZF0$K#)1vdDRk4j1BfW)~NJX5{j0QfnL)&%x^r>32qJbqZokPAy77Q=RpTm9WvU**eQo=9c2^Qc(9|)jz_PQQMN1wBFr-Xj`@pb1tBg89&*EbX&^>08270 zd##zb&jwvF@Iych@NhwqqvhlB!E(0A=(xPsLfpJd^nETT)LC5qQW7d1tdMFDh!Wl^ z+Ju!tuEvk6xk48bY1u(Rif1vq_nE{ zo$9&!Wle)NO4*Oo&Ky>UphFKd>`nd-!l&tQmWuTzSZ1egjYs9V5xU*EkYC#{_PryqY$K^!tv&kS zr57!rM;O=LtoOV|p!a+9c596bJXFBwYtn{wkX|B^KGv!d1N|8lapHOmK4sOH?un}1 zSxnHjT=JM!%GclL8lj1^x`C`ux&6IV+I3rH;x|a_YsmcN6~z_qWTkKN{v5sug5>I= zc+`TZ>}kKm!j^>DwuH+RYi3o9NqnddJ}ms6CBZ!wr7WY)h3=-?6>CS1kTAhz^Vv%F zVr`9;{H%m7{j_z)y=#*p%&sCtV;=ms?NJBEa^88t(T1U_6;dPPw6v|J77N5$cjC60 z@v$7bQ%ERNFy`rlU5)w?s)9u7dI>eHp%RLhy{RkhlcL5prO^B-ys#hX8!Ab<-LS0R zT?mNS+)<2|#N<}t^reWb5Xqt~H)-LMbE7JD=@vV-((9b2iNB&a%bk*RQK>6hp}|aLRpCfD zJ=Jt9M1U?U+-IfR+!u#u9$GPubEUJ6i(4<2p5$;>U~$oc2Ad!~^O4<}%I@KwzmJc` z&JK+400etE;J)p$(DKTyQ+I^&OjvSZoT*M{;~SeeinHeFd3w~af?Xn`gHz>1R(vH{emg;3UQ8CVH-7pih}({Xq?}+ zCDwmL($yvTcu-gnPNTUx=xs9X2t&_ZfV_CX$MWWn{En_*w;NLE6vl>r5okg-(#fWJj2M$W7?=1OxPBIH?a)N z4S-!I(TXB~ktdZ6WN_%xY9sMWQ9>b}G9^gbSsuvP35Fsb-yi*en?||7uheq0z z5A+(#3u^=p1r7~-I9D{N?0X85H93vB4ym(Bbh^uSy;L5FSTMyvMjw_)9|kMCb#E37 zCR^=q=K!`$9E1aO3#J5Bb$@FnaOYjYc7y3EckYh`mO=#39DC&$nwza+?71El;<;|c|NvXCb_l=Cz z&TXi{Zk(=RC8x0hwoT|PpNWp!-JD%cC*G&etv4tv^>N9{9^bBp2B+zd^q8`F8n|D2 z>QJVWv^OY%HhzBKugl*^o{$@P(0F0*WJA{U-m?ebfQM@T1jIDVR!gbB(vHx&*)bFs zKnITEZIaj$*+$ZbY;~bFQ5ljX)VN4cd|k`EHJh2LyJ=Sfy2nQxid~zL+oOJ9w*QB* zdybAQ2p2^^Gnv@Q#Gcr;lZiF4ZL`y{F|lpiwr$(C-O0SnIp?nP&bxQr^;ZAWYwxPs z)w{ZTtG@5|Z1%K9K>N=5D(FPlVmnl4yphWP!&7b5rZHlj>*K*pdxJ~#P>z6SPQN!M zu^(NMe=n6I`)f@g0UnA5a_-2b3WCM8UDfj8xsJQQWFmH(gOGKBkZRg*_L8)-!rv+L z{J&9B2{(o=6AtTMW>H*XD}ox%32~h=gXSvdE`LPDgRLlNh9pTuwI&521i4J7;Ee*& z18f|&_9-9HAzOnW%%sZS@%(xGXCYi5Um;AwV{9D5VK*=|j@2=_PWjL!owL1E3edC3 zJn%b-8hn^c!3xch9Jp_wLd5?32nmG}PdPb37i~y6!Pb_fI5A1x9jhEJT=_sMO(1I7 zS=gQCy-Y3bG&e8=qWQ4Oy$HMzUY7M5>3<`WLD-!P3y!C5G$3M1US%SpRw3f35K;#< zamK-BdLF(2H^VNMk%$Hepk7+?F56i5mU5SIBfW$bq9X7;#)eHt9S(o21IOr?o`?0B z4Bj+nFe0r&GS#N*&&?Hni`++vbp0{dIUZ8CjeU+4lI6-e*hBGm+Ji*%U zkm24OvtFpNWm{3Wjmg}Kk4dmayQwetAQhyA4t>$Jk%S$%t7(tT{>je+4kmavk(uV{ z3GdIl#~TL#rd`#H&rUV)=)0{bSaTYFZ006Mh-NJqG-=SPgvpyq6@2_C(rHvji<5Ua zbCOk+bZ8v||A14@-LMH8D3?J@$x0maNI_+$W>Pl^;wD=0ZM6Jw_-jWQE7uaZMQi0L z)Xj($rP)mQUDY(QMIzGarAKHN%s>_4tU+JOSH#?~p#ga@SXv>X9Ve5CEGCz0p z_U@g?<7RQr6oe_+&xEFqm9N%LS5 zh8&T_{SJxMD41(s?{6Vzrv70OvsG)1U(0X*=(wycd#Ru*Orc?RZCOZk#m}&l(Afde z6tTJKK3)vz?FxF~HFTycsns zHPYz%n5`0mQQ65P2U&1S!<-2xqUhwl3yfua{swmkGpE)pxbM;$=4j|8O=+-+e_OJ8 z@}9>49+7D8I76mZ#w5lvP&3eE4_Qh~(dAWx0T0FqJILFq`|%-tQY-mkH`cNngukW4 zGg)go8oE7KE`LB;QPXen{?xd88Jdwo1p?LocIQ;gKCVXSidbgpic9lXXErEvH^Hq{ z@I9e+;g_e7l_RDQT+Sm2Ee174w|&vjfVHCCU>C$VlWC;8M8Pzpt$}x<9;+XtiMMy6 zHipmqaSxFvZ_1=eBkCSILFT1D9?{rt7x&#JD_b;8dHR|V(v8b!2-XRB9^j)nYcYps z)7H1shy`eM3WoP*E1NO6p;T$hRkP;GTfB&-Q2}ouuEm}~h=d($humHCX z9ZSfqUy9k>1|#0Y`@AV2uS#hcQny-plPwzPXd9mt3y{<^mdK)BL3Z9`v%~hhz*3ri0%y`)X!N{oj zpd|?Mb?KdAH5?@~RD&@j(pnxd_KHo)=PGoSU?Y)mJx&jWG!Njx>A31ZO`)uK=zo}! zbBExG=yKeu=(t_Y!p<1Z7mjJj-z7 zA@l$0g$s=}Voa0%B`>X_sSx}1NK^*GM0ytIPx&-S1|Q`JBv-rxULM(J4(oGK;<+=Y zaQRdtCaj2T>r+C6_@7yfR{4V#O#7@^UNv4xnI~lu4~axh-~}$bB?xB+6Z!>7wYvH+ z=rCuV(N@6o$D};FgZpMeqq1>Nt{1D76(+diU1jG4kBOK{6jG#bD!=t=piU^~FI!^! z3rywK--#dEa#}C20;N&(>#bkb4| z`l(RzaN6&t8vCIPms8DX`^d2v{7U*1L=9Z4R2HDj<2Q&@Ac=a`!op1Y-HeEix1*R7 zV?}|j=x1+ES)<2G&vf3%Sy^mmsClWpsNfDohJH6$OlSsoq#q!aiaQ{fSv3b>zbMmz zEAMs4zKtKd>Ani(Sk%$;A4v5Q1G`g?lEu(*`Ec>_PCE9d`lNoXsn0MP)vY>vE%z1R zMwBz8g7uS0q2yFKpwD-6Z7A*z^aOU9Cp~L!a~20?Sw+fndL+~QiB$h-O8XV%jf)GS zDVV@lP;yy3wEqoX{19)gjNbip>w#fivK@4Aoa_w}PA9dA%-F%-M8FoHjz!lgWER z%i-y=PCGlGRE?cYn#X78cEe(C^xImo2xz7{ra4Pbvh3mRK@RfEcps{19;895rkWhK zIlMbRMF*i`&jw`{9{#-hc?aV-PE)~P%aFy)T({DvwQ!$MJ=f8Hd1IE%A(eHe9~2~< zkb`t@sxqh9E2)AhBsq*>fxe0qD>MPC)FOqr&rZFpETD=W{Y~{V9T{n+GJ}~zMe0KY{3tmikqI5~15{-n&=AR_D z4y;TbImoGBR|Zw8SCG)LPaB$q*#6QPG9uvOUI zR?6>0kmrnAVHkg0MF^8>?&#moF|KcbIK8b-|9)o_R-b?a}8 z`I$~Kmvu&c$7PcUFUCa%3NHrlJ)C-L-QRXk0(=+M_JuPv?!L*~`J(PlJB*AN7hb$L zuGUww2S5UOLR)68Tn)KQS;7LPXa%J@e`2J*VL{1YNFmHz*dT3d#Vf67-rVN}FRk%m zI8h>W;t9G5k&v3LMIqk>b#O>Erb@2_N7P#8fPui$^dY0!I=;=d+8@%n^qA1RVU~TD z8FIm>9kM85$F3k3RuSw{^UTyPu?EVx$z5xQT~m|B9KXFrPv_gfy126}d`;L?s$<-w zj8b4wh!hm1+MRVTc<1?7Sd*IG2=gNyUX*vwTSUYv{=_W+)y^)qevS^Cz*cLZ^46I2YNAgOx+V5I8Z0c+?Mt|-ofoTh9s?o31^5y!LEe*Y+ z6~I*{pv9x5teF8eoy(G9AIr$eO9zVI!aGf9{Oe)+b^K`y42V=isN|hCKp{_M2rI}y z^xQiH-ICW~>fT>*$cwMe!+cH#&Zm^KY*U@^a`u|IqlPLw-^fg*2NX6&J$61^ z#zZ|5PPOK;73LXnMw<@x5w~yKPk)q4Q_VfFha2tXL$}62RD6SxH%q*zb{g?w6q#FJ zBx`9Y>Zov+beFST+urBP{0!%%6y^nv0lDdB!)(CQ`wbjg+&BZvCe!i942%ne&;pCx z?y-^liio9Re>ZFC1FjZ~jgQ2n`{Q@f5R3hDJOum9dzsuqmQsq0EP7K`UR0}Ta>6(_ z&SW2clO619rN`&Fy#b1p1p6+m74&ztU3&$SSz2_KcM+S;vAf*96MjTnV60TE;k>tB z1Rrx6@H4laV>cm6eI>V5#3?22gWk-+B;<#|(e!mo+Nk=AiX zddZ*1N;LB-Z$mGEr}dsVlKgOMEF~+o@B?!&X?uZw`yh zWyZ~Jjzjzpm*Y6%1iN}=&uj7-fZ&`AuN3lGZZ;O1T)LjGB~LDQ7>2r2ZE?Wpi|ydB zUx>#V+B9P$J$0CuZ7$fnnq&wDfd+qh!!}!9BL^$hj59L9wR7KsKvxHIchZZlvFsrIjweO@p5|4HiIU?tf!8- zlMvH%-El^L2bSyUmQlNWrz8Jsj+Iy=G2Hd z9giuBLwDlsn4I<<+qeO?e(W@{p?PLhG&S)f5k3qXz~=Ob!G3;pwPY7=((LWhcPok( z5owfh^B0#R$r!+CEXKOg+~P|vk*C2h5DclV-$kW|f%}C>8!=!_3O`CjH+_fkG$XQI zFO%=snZOQt0L?j}jY?}BP!#x`s4uN$u6E0z7p2aws$h5)wZgzbHjn#Vh-gL=4aQXv z_lP`%_)_F%0KMicF$1NZ;pW;YR;qng=}GoRFMN=7E)YM;%?puKJ14Wrnql9${m!?= zhB*bD)9oWt4*k6OX&^>Swew}_fS%9pZ|cL-W=A1O0gwUMpRE$(ntxqp95Ak{BG3I8 z^$x5uUdcLNeVJLi>WnDkhxEyimz%+9@d+}=$*OT7 z{ZsEo9J@`>7VfmymG;h%t5!+D$C_G(_ea2zUw~s(sx&@I5bV2&C<_gC0SBh;|B}UuIwNEQ~!4=wPOHL2d~?r|fWj3Ah6WsLBhU5J96> z+VV8t_O(-&+c4OW)c(~Z1r!466n)K5WtUgY(Tox2DhWHe*TK6~#+FTyH9?C&&g@T%NOQTK69$J;Q{N%BD9;uS|L zg$qArD{2up|NWudg%5;OIGiq}Wm9nGr_;8srkmRpCbK}GzcreQVT=+joSNMzU{gWO z^uUsEtI+aGvDDpXi>DAM6r=#fYueW0>p z-2}y9{rAW9SXEjnlqP|T7(BIIhGN>d~mJ+s@7;r8mh}vdS zC9TNBIwX*}+mbVcJp)crnwquc*{i8($4l2ShGgA)-?leRbyt31uG{SHrAdi8rOgBd zC*ET+l^eYURRGlQQ`Um()oWWXp)~+c8y%S*`ImORi(bL&4!vSKBq1XbP7w+f9%G$y zTvK#pBU;B+jz-(OFzSW4c3DnC>+3}gX&16M*xEn2W!&Oc%qCfi6PE<8vvPtpwZ-*a zH;y)(a@`BKcQ+3XZA9;ICj)L8-qUUX3pkswfj4u6shiIdn(b{ZR_+v=K5e~rk!$$47>1t76V%b_Fu44Ozm8vH3VO~E3EJelLd=z)&c zSNl2pfHP^!83s|KxT~^iUX@)MyKt{~PePJ<|9gK{Q(wa8!F&O+%Q2qg>dk&LEi8%fFvWC@AW!QZu7;C-kF!2E9fnTp6x#1F<6&@Dk1?V zz7mH1R>Wie%+d@l|A5P3jw@g0p7zmvt$8(n^SqU_LCHPLzS`36RPBOe57W5aQtOm% z%k;5oUAMp^(QDL;>OG(S&=JBVGIr0bXN@A`Wnz@@pnhklzSy8Tb*2p+Qe!0qQiFlq z6n`33gkUl0jQU>7n_OZv&$W@Tdkf#Z!zG{88(0|81y9p_E1srQWh5J%0W(B4Hd?8- zkiKAzkH`Dey<9?miBn%3@-ScytnxgWq{PWH%x9-^f664#eDtU{(P?4>uv$HI@)F>K z?5;3|t3KB+OYA3o!*^kB9~W>mYiy&vki23X6^YEF2Ag)xD#l?k7mWmcDDV z&7b`xslBV*tKl}{+I!(6B8jtzUJ0xvWed$O-v7A_Bn34wZX_F%cdLBswL+=s>u6+B zcO)GKej#u@?#g^1{OIsCahT3<7Rq)3zU53vH7~yRmGl*X(nta4|LLKP)ejc9Np2VDhzPkZ&1g2YNEVs*W z=g`U{!+zm-4BN*&=s0aGw~xNXt_+;!9@TZHzu zC(h0BwFQwO0v!9wdOdfj`=Wxqah*qYT6UbC+4YasuZ#LTp3S|ejRpb^F=2RN^X14)3<1Jqx=+v28$9UiHkCaM`TG z7E_It*ijvEfM!pcsol1TEJoC!@`|Ehz3WW+OLRCZ_zz`hDg@OIxiV6Nu(rf9weL-w zL@wz^S-MiEM{n)D-%1iK&Cy9!%jy!xl}+X(`uL)Q*|oZLmfiZ`Sz@=Ogm<)iG9}l> z)6!Dch@R9JnGNYtKRz=^O9!d&5^cF%be1KtZNBvKdX^>8gRK5`@zfF9b*HgDT~wBz z7kAvOv^A#CgT|tCcPFdX;use*`MX+c%ASHIa0WFG8+Dv6>>d&`G~EuDCJ-dyrBtrg zTK1?3Ji|3K`sTA<0@0oX4m;{vTc<2bo1&Xd`&_xvSrD$+q2hm_QHXG;EHTvz1Cm&QLm?pL*URc z)yn=h%3HLs|nls=Q#ciy^ ztWZ6Y^cdy$-9Cr{z~R`@!BZ+Y#?od~WvzunY}`kd{$kg%AcBgPrbB@VrHDERhwfZq zACUNbe_S*z=fK2{fyI*vQR|l}pR=^175uL1*$vpNH#1cU}+uv3rxXh<$Y4W)=8DiD8>aHV?s{(;(2ZW6Es-IQz*_{l|j3Pt=4 z4|O~Q#GyC$%$><2=UOX3v%9!)y5$#iq?y^pb-;>;Et7~0E1dkS6_JO?}*J2XA3pD;Rf-NSJn@=h;ta?&}8Z z6;_PVh9LARJGpS$b|6KF5>*`x>}g%ZP~?MqfD6dW@8xNA96kLNRDLv(aWPoxuS`8< zm%B31MWiIt^&Jks^NU_S-%xJESda{LFBTkj=wwWJZkmg5EYtjZy>B$S!eYN?=cX6l zH85$X9iW5@7Eui9DgGJU95@M%;%&cu+kNp>%xATT1E|xaz`izVlc$@tHhtmbI*?ta z0#}A6yX)loD@yk_8j4fEIzXtxlw`Ksq%0?kG|VZ&?W%=-RCm@i8gYS^KSBp{n}qry~!gy%>l!w>iR z)l)&hflDcCNop!jv7wZS_lPWTQ^f<9_ecBI(KX%?ex#&ER)0ak8L1A}q-{i~=T|zs zWQ#3z)p1KV4 zHZN2s^DqKW8g9DRKgJN=SgHoS9Z%7Edk(U=T$7zfQ{6mH(QG*1Y~KdKHr}ISVHL>{ zTe&%Dl9#$}x>43)kqUJ%L6IeqvIrsp#&_MEm=kMy1{Kkq@Kj|CG zdYyesR1qXn6^X|#6u+F8&y{?Jw$+!o@LcLK_>F-lf8B#tvDi1;>!IQy-8OT!!y(V| zI-OCTB#q*GNhmlX@btL7o{OS;2FHBIkb~@;Dk8i5=s|qRdD1z2!h2bZI_bKF`#6@< zdxuHlCo4K5}BL6Ubv4}5Nhna$3 zzrfECelK=xb8kCh`m4RZF}smz2(rI=;@Nuo+jd*KXSyrr(WRslr?hb3u)Ov}hs=I$ zte*4sRC}%%n)VdBAM2{FphHOir+rp!z0^MRrL5;dABH4dx=sN zCb=yhDWaX!{J6JNwD{{ib&spF6On5bXW3LN*wkw4TcwDr4Cmcp_{8huqV~AY@zaJ~ zlTM=TApGmm1-D1D{aj|nE$*Z0ROGfvtEqFprY@R&w@6l2GKRceh`UNaSr!Iz9w2bf zTqz96ei|IB)lf%-R~b<|%AUoM3Ane<+vS_O;5%@=6S11UzP0PZ@WXbwKDCx z6Z=Hgfy%#2x5?KM=2qnzm&Ek}^;))L!$-1Y&vdZc(<&}ymIi9x|MQwl_`9N@PBVwS zNmYs3bHSpnXl3nO^HBRx^R$b6+gx7L4@22TwF$Cmib%I#X?$OJm3fvZa^y@Rb z7bl7mgDEXtIQ;&R=W#9&!?_kF=H0%u27N$+F&* zE#KD{RT1ULRM?#^*A{s>LB+GPzuUwzFRW6U$$@>Hj9ydjTkfFVi+80dS_WA;-^}-Q z+RvML>E80&%9T7-iJUa@Tw&B`9Z++^0=ly$AH*8)Fz zv)ABqIJ$)5T?AvXIq}J$GV`{~_5p4Iy`(36yOs}9AbVLpxVT?|VVTHg#FuVc9bS>o zc^_q@uP-$(of3v%fx;IG(o9%b(m~RXc^8^{nFN z!Tz)b@lZtz>--`EbSLOC&G^ z5#LTeK;TslQS;hep=gkdJPJQ89hdvSh24<~3~7j9?1M%4Z^y(#M#HD$F-TcD%1Ab(AbKq2{c3M$D9lYS>vr#R}kYg0*v}5 zYq=UDt(F&K;Mcfn^)Mi-_Xa;+$JWt z!+K&7?is~n8vQvYfylyk{ZaXq$?uxXE|IdCMn#I)a+?y2q2uhgjR8EoB+7IrUUq|8 zb&D=v7MOLF>JoAVPF#8IGLpGQ~FwGG~ zecnh6Sau=;8^of~6Iur-jB0`ztt@xmei-Cr#TT*M>eY6J|FjvEhie)A$@&_!7`0b4 z;f+vYE}oN4je4Vb%@;W3+Rxitj8krdZNE|vdjQ>U%^1?ZJ(4)S@<7vVo~zkEv#sw!5InpNg&}trCtR{>>9~j=)%ln{DbL0aq`Tv#xkr!K`+i0 z+~xfHcn4{I-Y?O(b`i1z-8ih4C&a%2=f^`6= zuZdKuu4G7FBacIJFsy@Oy71!eSw~F52os(w$cj>9xwG5;>l?-DZxl~6-=;?_FTemx zg}gU=_@i7yo1c~8Jxpif>*#^q65-}n0N3!gOZCur7RW0RfeJiX4bIcxLmJ7)5O`x z%0B9(i4DtqU^wM!9xVt_Vfft=<5lq*1n!RA-?aCeUo{_#`t`S_GpCj2ttSKy-n?)E zsNsTK`Y^3-90*5@o_FkCV%5ZKF&`Grn8mA>{$Q252q|8ur}>hL!LmHe7HUZD?=@=R zW%y_1y=7k+^Fql$6e}^dDOrMk@;q$yme5Tv?M$#RDwR)=_IQ4D;)0BnZt$(Il>^V=!co_&R z>~!Je!U(sHgUaHj(;cZ-hAo_T=@9{aS@2KMq;@kZW5Wu=GRGn{QF_u$F!x;B^ikdy z98GcrH;nflSL|}X&1fShJ!a_6)u)X%6L0ImdFy$DZA|P{Qbn$vJXV>_A-0l>eZ88K z4&g17S?Y4_d-(@}kc)EPDiE4AkBN#ax$G`mP%8Q($ydWz|F-)sRtKXO?5J1$(W&K&X^TNt#P)lusI=6DDnR}_c|AX zOFLrGL21^+fpVhty{XdK8t^m6nlV_cyu5K>7(0cLxRWfSHk?Ex7Hg@Ni^b2uQ9KMs z-H~IVuwpXVO(;x|b|WBZ#A$F^)IoyR*o2mrE!Xru863Vg5o^q1@lWRL_C|9e{;YI1 zQ!3-G==iY7TuOOi(QYcjvtteNgi-N$yvcz=P$GK;vYCy5IhU-aB1aAwyjc|(b08D7 zZ9dJM&nn{=hm;#`!b!|Lnv!E^VXUi|_1k(eR! z(c&67Y(sG6J{pS+-~R|pVH38+WBH%H9v6Q#kRE^|GzeSE@0|@V%VCxvW~>z4s?Fa2 zJ`lr#N}&_BB9k~1CKx6tJZ2oxOe)t-Qhf3qSr01XK1~e&&j> zJ4i4Q(Gv@Tn5BDjx-k26(dzY$kS4LxN`ZqU|2}!M}*Z0x}+e@5d?X5PG2LanH zlM368$CyplGq$IyIThRGTAPhRKpLb+tQkAo$3lfUR%zSF$OEbce>$=&31RhB{i&y9RAXEhFC>p^4eglYyGy?Eeh5A1d z0mYn}dNAQ#6rl!&7CEcQnZqakdI@gm<)!U_wno-KXFi={@cR%-$e@5qW3h;2PNPAS zM=dbQ!f7FMOud#1pHEAA^mxWAm>m<1f@#e6WgpWkSXfSXXbbFA# z;%#UMQd?c~+$RwzR?*@g6vb_X^#?5@;j^B!n4vDL9}B#0AP zTXXY>v!k@*R;aiU^O?gg^%zqn~Z+&KFY-{Y|fHUszo44ekmP(icF5OZHh z0)6T7ebN!h?l!JL5JBGppWZQPW~2W=(Ap1S;6ma}abyObNen4Q4vpHKU}FBr^bTu| zFs~0cUq;GujC8DV%yaa=^Jo^!Zj7WfNDm|?UjKT@2ib!ZMb~S@z>~)T^`hGb0ATzd zXbINO^!xvn$o~J9xt{qm$Nv9-JYv^u{2Abau3SU4oH5WOV0gu$N~TqI4Im>fzM2_u zL%UE<7Apu6MigOv) zWv_#xH5f}x_I{mA7f8lfhBrl1k#TJ`13HMNE+M7%?iVDFlsd4U8#a=;a)Yj2+BvObIFeSGCWDiU2!1D`V?_Gy}uu*9C@Nl~D7)K#PCI z1~UUA9TOWTt2QAs+b13I32iVle4-1V9>yo*z`@A&pR&h4=7WWkjggRzoso{2gNXu$!c(XmSJY1V`pUHAY|oWq+?}a{;bIOxq#!}HdxvJ zQ+D{LWPG9#te=DfE9YnNPf&u8iSrYC_$P$?>*c3eL&wR)$oa1hC&TA*CPG#=*3WJK zM=Sr!AYox*r(?I2`1Kp<|!AQu=$wtTWk0<$8>z_FB*#P@zakkH1|7+{>Nsi!OTI&$@Y(>_-``fKmCsXC1x-(eeTxJo$>!B zGZ_AFRt6&j105>|E9WP^!Ny7VS(EwGb7A77)s(~*uQ`ENfro4?EaLG&=z=2hNP>p-XxP8C?7v3P z68=PzNBT9LY}VML=ZwlahFND3E}>y4IxhWKY=4`YER7-)zGgH-)2U7t#yI<)QQEo9 zl=|e^dGvP2J9Yoyl%WYIJ)K`EQx^DA-3^1JFW@O`{9fcZX!GsQE6$fo?caP|w?m!G zDqrB%5%hTZy4nkLc*^SM)Ziu@78aG`*MKXJ-}*fe2shMtz2(Q|F&;aF_g21;6<1c$ zAbM})BVWjU=RE;Tu4SFT6+P(NDPP34lnq8sd^y&E-^OHa>a_SUMC4J1Md~CF_-Hk_ z`Kbz+>ZtraDYoqCUx*2TL~hbcJWwgtkIc-PFbAJ2(AfsQHzr&S2bm@#A30 z@(QjHh|7|);Q!0?;2tS8Z}SJH5Kkn0A*5K!xHj>i*mX7jYvy;VAAWG(j-w$I%`G}X ztRpa_gefy(<3C%m{jPRUcesb+Kn$_lg09g0V{$^gUf*e19x=gcpx0R*Id<4l$H`q( z!XYzWVMzId6x%|#W&Lpd$f0b=nlrLP^pNvR9ihDwwp~*VQEYvv3J~{4oTCfa^&;-U zrQ_Wd+_`(^szvDvp3v2Z;Q}EGaI2-Z6uoI5{(eJfNnV9#kY5$OK)2w}VBCN+)SXC!2L$f`arfQkUWZ$=pK9p%TE?KZ=_Zvq&<|^L3LL!&x=Z`+oewdytPWjnyEm*j@l2>GB)Z2~V0y~|d%@`*8^f{WM5ud#|oqYsd{ zGqa`d`HgMqD^b5?n}gn1=Xh5)*RqnrJ=OnW%A*Z~8TF(K)vt?xRP(brCc8GQDa`i< zfd5C+=RSxXnSLnWDVdt@oBBLHsTxg!P((W(3@1q+N^V=)F(yx$4(zuRW6Yg#6Z|W` zHa`L>V+Q|gECmR-1VcjQRk_C)e2_-DzdN+WKGw7xr=SC=SMo>7Mc^T%H`<=xNdP}B zFAgYU6~;vImNB`{s$Nk=nlBZmZ(e=7u!Y#nGwj& z)iY#20;o;{XW8n&>~uZUhl_$ofUF{{ypfz|$_CS#KHKPdz@S_lfzuwBA&5Q*-Nw|3 z?uLj%x;omaL8Aq$5M`VMvuA(OXw3NNz_c^#jdF#yAuERe!Qi9cDs{5D)>6*Io$;t) z{9}aPY!c)9TQlzPWvj-m+zXU#oBay|&bigf%Lu<3^{A*jua%+mgjH>sajl^#ah9={ zVLjmJW`Eo3SH-@+?>fP-4)+=zCtw8DAw;#-a9!XT-T|ebsbiEKe2Eg!12C|)ctbHl z8i%Sx4fSiOML>z-WR7qVwv=DjB<_Mk4J5io3S+H>z>Z>_gKFy4syqw%CB-PV`d!uG zL)IsMJfLQ_ZGQ9n5{$6lQZ?TRD@C+ZvK(lW@<$KYEJUb(dDK9^CXu=Z9yW$;Y+qbezx%1zc%;Q)2wTWD|a!sc0zkfM0RHm?Ftj=k((6oEFul^E>W4gMAK1 z350kwrL2Qe>P>^%GhcOxvpM5OP~wSLyhD)NLd_<1!__hcnQ~bZekKcDwYEo_IW?;k zsBB2TnFtm3n(>o8<1;O2zN8u!@Z2s z&@iX^G0{Z#h>^z%ss$E#J2%9?Ew}6k596EBhw{@acz8#x(#~JzJn~w|Mf0=QHRnke z@VEvYL36*n5Ao>+8a^8;74hqw2V*qRY~7{cz+}2CR?Pe`_IT4zf<`LJ4@EU>=0rZ)4eie|Zr|w#0@li)R#I zL~AUipZ!R%OF)L5gQ_HHmVHIKMCS8L!$j&#Sam}`Q;#uHl0i^GlsBG_5b$*rNgOt( z15IaBjD(h^jH6p7(`q$|{o=~a&0wtA*sVO?B;HJjKE6zur{*jGW_6alzZSt`CTB8I zINGT^M(3y^v{^huO%2O)wrsY8{ALVch9YtJ^>_1gM-&TXsp(Sq_D)ke)eDRd7}^YE z57tXhQ!Nh{CUt+Sc2(!t%g^l|CXz~3X*+K`jH#~M?a8oj-4-n-G&NSVUOaMNwx3

I;< zr0e?7u(ajkxOB;Dd-49Z_P60G1NhVh`S*FdFA4z{^c*Lgh0o`3zic0>Id+^c)guhp z``wJ6w8xA2;}Oa|uy6OlAk8K1`OkBtQ39DbbAiIiC4STWbCMKL_)Ll&QArE!AY>%uBxE2|74QLFhZ6XUl#z$=CnHsjra@GHX9#$RaY%wh z6bWy}{Zl?Lt`ns5tn;o71)&~@fMa&W+-tM?MtI@LeBl}VoE#7*^K8ElwcM_e2rHgJaJ9`9CIla)yrIs@z`C*fJ3!SmkeerOxh0a z9Sq#*(j#aG)9ulEhCB~``iy<@+uF8yhT`)<%)##TwcSx^hOp4X9}m`0`+}2=fG)(# zB0$X2^Yil@=bBZ*gp$VBa)(O|;^G>LCiv_G-4QH3Nb&k7et*=Ap9-8XPAJ6=xeFBk zKp#b(-%fEJo-&W0*bOHBkoT@Brf&5sXuui&jc z#Ao6USnu8}30BB!3=o>!o~;Xn_fQ>XgzS0umq9%m=RS^Sa*rTYJ>V^RH$EnPHX&3l zA#`mv6jdw3R|*CFpRM3B4El1?y*kM>$>6X*VXiqnd?}Z4-t=^Pr~;tbh_t4`+mMKU z8Bjg;eI%1{1W*uZ*ugLg(SXn(w*{n!;GR)rp*FS&af5}gDRF}guNiR%b~wKAa-fLm zBUtrn-5cN^8h(u&WK{cOXpb8?1kHg=u8%A`NXda#)YGRHPRfC2y1l@G>d65sRRcF> z&!)1CTi)NI7PwH2Zh4KJ4vA<$J2?cd7W~kRgj0=-p^r*8XkyhvS{+e(vnfRI414aw z9*A6xr{kk-N8FB3F--*L^S0Bq1Ml)}Ggx(pm8Iv(4!t{$v%qfr*bl^GYcp9KlZo6{ zh3$MWO|+u!&ph|~Bqei_Fh7@pNw8Oawxf?;C!vU@jshN_RK;C2esHCHnB>+13-ek7 z1abo6vzFu!f%2-Gfh=CCSmch{;wMC-Eg14{Lh+e&lJnYqdTL!tNlO>ES7U&-Rr|qw zlwR8@fE@5e&fWues6Yoq8m@zaViq)pqnL4aYnU>2Hl?YxaY#E>@X)9+&Uu+G>DZ;T zY|)5hK{TGVTe%fV*%w1B6bd|6`C8A=SfML@n>ckNm#(xs5g{d>Jo!c#Ty>!80HV?z zQ|@yKQm(8hscPIw4P)E4!X%rwosW6F!uH zl}lSoGGS=w0BavFP6=S!suzU8=BK?5>pmKRPb%>j3v)DbnUoCss^8az7~#YN8ag~l z`2s&#hW-OBPEzQKE*Y;0p2ys?=q-?m?bhN*d|rvIHrFxV(k+GCw`5e)1f;Hox2Mav zwm_edKlPVnh5#G*GN5BzM_sC}sk3r8IRQUnEg7;EC-tXjoudqR=EIUpPoF0|&%h{6 zbyL)G_KxsWjPw=IoQz;WmUK-y$kIhTglLR5NaLN#o^*3?+n8_=glS~iLY{CQ5WZbg z;jd*sW!g=cJo$Vrb`-7P=DH?%IQ}q{G%!K*mVQPRB6aOv824P*CPh`mOxptRzA~)W z{@!lJizy-!`;aQ<_o9So8|DwV zHa$Ey=wU!yo;m(sEFqA3g;f{^Xn^RT)i*vUu7&fsZXTmP8hVzipG6n4Wkb z=+%_iFj-XEShBvw1=cW5Oeb@}as~I3+-jYonqh4Y)MM`5_We=~X`AKUkYU?TQR|%? zG@5wOFc*bhYTSvlPHnSuz&Qh$2Df%Eu&p8*$>KYtpZ^ie+2jf3v9@-#QPt5@ z*K~Ea8?gF}cWAPvrLih|O02P#5)ZiO!B8D|x%+>Zd#mU;f?&bYvPBlNEM}I)%*@Qp z%*%b!Y=D#d7{#h zQhS@%OD@aXX{?^Z4o+w~86_ynW`O$X28~V3))hD|sm8DS90W-E;H$qIRVi;eG{*bOYZibA2FEX@)J} z%K^=m5uBrBGZ`|M+Tu4nZBFeiLVN?7Bgv3c5~qtWWEv0)%3kz6q68WRT7m<=lOsiw zjFMcT1lJ87FnW|-8s_DX>?rmf$hx!fZrth2)5u=3;n^zb!ngkhla5l*ztq(4eXqU8 zS?cZ7bF9Iy_ZqUAQ;u`jjk{m2X}242eg7M)OR@ZOve-HJ;LguBQ1`w$nj8Jyh%DW8-3ed!Y@Qsn%QiNAQFI#EW*JBnMzN^g!Dvh^ z=GoaSbj_mM?{TS`);@*??RC8L$PVwz{Y=@wm++dn3j5zXO_U2dzrBHlJ=Ja{Ub3a2DYv6ds0D)RoW-=T64>~ zXj-8{vK6Lmk&{-AAr;aVYYJk^1tKxF0Yg`lDnwluJ<}wVouLnMh^ou;jAt{suH^ZF%iXW!)=@z40@eryvwFoRY1 zY3{*TVua@l(aSf#*0&3KX+ThzC}Nf)l5~G9c6G__Z&o5;{RSnnhI%IX{UI2g*rgij zI8|0!07{g*HCG0!kNUD`aq=qE+$kWt4q7P!D}--c!8DplxI;{{vY85zG7DWfF`LT! zpu(wSgLIpCn}VMxYY@#kLUp#9aS6>jN)<(>FwhL^Mukzn(1&_)!phs>!%bHm<~wJ{ zrq#jd`#FX*^z~qdOzSZv}aI9`etYA6{8*dU*T#tOk~_E()?bfYZqlG}{5T@hvu zNk8xyph`L`pk|+Y#)K90$+sI*IDL;c%dvp~-@DH_!OGuio=X8o zgKhS0@SbkaTiKP~T>+-PLGN5GSbS}%)F3!2Pf*A!ExhVf1(~&`R>M+tixo}_?;6;7 zu-Tl+Xo|BcnI^ITqj3N!l|nvs=J?%AbBA+Fbnx`V((IW9|0t8 zuCL}px`3B}G6eU@RQKFp@n#1O02|zm!wg*Urd5`5qEGObou*zsCPrn zyuQY%GH(sL`+i1whH`_r81!}+N4zNqdC9}zJ|Fz;8*4b`0x0skQNLqR%r`|mF(lpD zJi*rspe=YfbyG(^aUF^mE(Z$*vv)}EG-8ByRPLZ9hR_n}uaMv(sUQ`KuAV|OuZm?& zAqN^a(+2azZA3-zk!@Rw0s@1VfS>E2PW%5GOKR;K7RzcZC>#X7iOn$faM39 z;t&|lHulwXcW&z|Y^&BR@kxhLEj9On38(aQnrS@nPGiWVDJr1|`Cw&^Jl{FPquKyD zz%hGnh^Y;0yCQByGc-978>L>Ii7{6$dmS$_(!xQ`3mF8Js^clUGh%VZDt!DQfNICv zRgu4JY)#@!;GD(t`lBabb?U$cZs7z3fa_!i%SyH{L>fh*!G?sjo9JHztMFlU5 z?KBF&2No(Vh?p0~bM&YRl6ACGq~7(RUcNQ*F4E6 zI|KL21G6)Re^6~-!aN9mI`h7NtxwxkT`qo`0Ha8u?^Urpsm->gV8N< zvV<*oT7_!ajSAMKt4j%bnPZ;y1Az=R#0Y$XtXQWWvCx2` zhr4cni#xY3p{h26^P{12J^;vpHt<31ce6Z3Zwtr6?yPPnFC%bf@BvP{|BZ|^PGawf zLf*ToQEjz6T%n3qIk;ku5+j(1oJKP;zKl=?UYtmuIu(3>ZE4{LtLT&(>;g)QmaU?C z<=+L_O0tC^Jrh3v3;Uhj`R=u-6z!Q$SXyERx)V_dd#JN^D9tjUmYFNi_au`?gJE;a$nXn2$_x;Jq z0Xx+6EAAs)816GnSI(Piz3a;}iI4XX-n@QB*tJZ&@Hc&eWBq4%NoHOKSfR(3&^xHu1wh`>`zMa;y~hto&Y#ijwHUJUZav>cg)ZcG|$ z3Y!Z=6)o95qdGE&x|}VI#q%9sH2_T9gB9mO`bVM4+-_Yuvu+SB5wI|uYDTLSHpqca zT})8$(}fm7(?^p|m2-*+OwIcyOx!TUL7Mq#8xaYywE}e}HN7VZYwP%DwXM+voOE@dxp1l~^N&d%lrzlY*yt($GNacE66S7O5Dq)JYtVNr`z$2E?{ zeHOB;Mwy{fwfXP!b>#3e31(3$CQ%GolWdCsEdA*TUD`&{70&s}O=yNWMK{NlM$Un{ zI9Tv;_Df#k4ehzad612QpQpwMBAGbIeSs;jK;Wk06=4zF?Z>I)f(>IUT9!qpTJw4} z5MixhGu9vuE~|WiNwh7~HH54rPNQI|RZl`|*W1`jB2%W5xK5+qVP&CW0Qz^^Uf#Yt zhm-@kZg%%22y}e69=Ht?;o=2fdu`$^YT{3dmNoTw>1P3RP@MwcvP>$LV6ekWTqiW} zzS541(Zwil7L!z^9SttJ1Qr(jKKZ-TL1qGbhnHbmw!bSkx#4DZJ-g19MCnJ+)wLXG z4{uwnd>8%;SDwE%#-~xob^{DmPl_m%Ycf)nb-_?#KPUu~ie$i&lBBDY^Dh@D;d1G37>M6nOe$;sW~e8Yi^T5$8LFDEYnnMIyM!<@u@$vos1wjRO&vMqhLI#% z0A8;lsOVhE0rBENG1Tl+64}3rAgWjmY$7BG{>*fWi2Dmw#YLdGW)w}6G?TW_YM~$p zuIXxzo1gZD*z>Bs2^>O_OJ2bxJ>iG@4Mj7AoG(au2P!#Gwhc0KE!d79nBVtDfB`xH z0R9*^-nlWwuIoka7TndJoT0#a?efg0K3A0cvWghcqNdg4_^Mt-uK_oygN0Kx52y^HnSD@8I zvXjJov`g>JI3MDf!lWcHJ*#mx6}MRLl}NO~TXNA~AU7 z$Td{#oT@%)D-XfiuS6Q4*=PZS2sjcijx{7z=1e-JPRps~pk%LRi&g%lZSwD$&oX$_?cp`vA+bu}q4(+fWpim3GwDW`HeZZ61Ttzv!B zcZr9=$US|13*!|wnqT2f>2?o3<{FTwkar{LWF$giW?&NbIy>$A435G4t)(1az!Wkx z?{$c1W4_^6WI0ao0p9X=_5>4?NZTMuCc~?0bDPxAK$}P=vJ4StjkbC|!@~5hKMA2# z7>%7dm=MvlNP|=E>>4x*LTWV5x~ikT6#zbthL9 z8mo4YLmF59+RGdE9rc?w75Re*`q0l?JANPcRXazK5%m6}*Gp!nK|bA&SvwAw_ag2G zf80=`GW z-d)Ss`JWkP{-!%9?K`@$RSR_mV|`>2!IvDhioNJB*zMrmv(t{Kjty{4aJWQ}dF&2a z;tW{r(BH1w1LiY@j&!h$g5kcY_mQDtw?pyF0*g8p86$TP`8et>j*rQaV%8lduD>z%JhcTl*R%?`@lTYO$I z4~{wO0_I`Hx6D2G>^tj3&aIqlT}2W|Mk3Ex;?Hr8r!xSES;$mS=0Z`Wlyv@&di4DtMj0_vVun3S%hLSstSoLEh1Vsmp-SOsb zRX!dQQ+|Bw6rBw=tpBBRcEjR)avt|E4z!P=lP9W0(?{7x(=WKJiT~!RfG8c_8p=_9 zs6xe3#NP7F5SlmW={NSNHQ}0AU`KZ3ludreSy#4~6}0x3H%k-J1ptuCeoE|JP12UE zCF<$>^T!`7a)6eMo-7Ntmiri)XZr$7kwx#aZ9=+9&Lyx>u%$_pj@X@gEB5gn_*_pt-IQdLrx*f@9pr-2xc=(&7?s83+^$M%&Yi*D^%WY7##(nh zPsFfVchWMEf*^(cVm7oZqZ@AMV%T%rE-Z{ekQe7s?eOINE3l+q9jX~W7p_w1>NdTC z*b>xg4>L;fLxr$;r(RFwt@Ui8KHm#CLX{nb6`ISj?8)4kf5*&dI`kjSHjW4!vuCW) z9q@g=X@2s#6vMa2aNK-VWP6LeV@C8VNa-c6oKThn+NTlF@2K^ri|6nca(@i^_hq2b z!>83II~@}z|MV=bDi*RFC8*{)@K@B_e?!bhZ$#Y@&^m~-SdrkBJc}#lN)^+jr7Vl; z`v4oCWf=Ki0Ez7X9p(Dpv0KcH|0BAko49Ee%zzYfd5hMup-1~JdL9TJbsmWMR?v#L zLk?t9_^>~_0~GN-Tl3fas0ES-0|>PlfZbZg!MkfVJ(NEXtUv{39edsVeBC{2+IjPcZ>HpjxeyxrisPuG&sWLNI z==DNLI_DX^+$!LKz2T4pdX=aUMg*4=j>j`89TrbI6(*OHqDbr^SByr;iNGuQiBL-y zh93ViY*Tsq34F@}d=O?)_4tHDC2c>C6Z+3m_&-ZYT^M>% z4;OJ|7eg1*f5(b55HkMLy3&8}od4Lvf3;$twf_Lge>fr|A=5u>^4~`$!vDr7exjfM z=ucTw6H7xOdyjt*=s#4GiIAO>>GK4_pS^rut25z0F7iJ*Hnsc2O08i23yl2#Khge! z|NM`u{XdfJAG`ZsW&1C#^Z%u6|BtR1|8EWcpDODg{`y}9`ltW@0bTw7#t#2yf&SUy z|4U^t{x=K${44&`692i;|0JCAKht&n^Z-^EdTC3OPk&=?|J}&R$imLb@;_j&DZb~Gl-+h)AI|8P@Fu3Ow!&1TvT}ZHxW^F5 zYzh(uCvU>{qq$hx*p}Uf17;1atv~&vtK-C#9)X4Y{+P#gYKA(8ksJ?$1YL)_rAQ32YUZWg7-K5j~md3)7y33N1WJq^$Pzxg(E~3 zz{g&}br#Bt>&IadZ7|AHT9MwRe>cP2YXJH4NblRh<%d-9L0s=!U2m{AQ4l0d%iDFE z|I37bXKnh&ErI{b_GNEq)w_;5KKE1#xNN(hP?Trcm@1$-kJ|E`+QLY|`QU@t@Gq3k5dP@EIfbrphG=vTgMzmY<8^U zJ^;tIAYbv-C9^8jZ&DR}s@-odC8#T;=p$LDiCz8oo;<9j$KS|+){ceaSk!U(=nAi8 z3b9OSz-R}`l>szv3V^=kX zpP0=QUfG;F*Z3bNv}MkR%3;a9%3(p;5e}Y+c)Hog*J6RcD6gYGW$s$y79v3@r6TGr zF}GYFLB%KhtdTR)K3ow!4}zf8oT^zJ;A=AaOvs@WiPuQ3)$56R&BeP4SCpGyAQ?=i znx7H}{~uz$g9ZA+@3rAlydV3AekIL?dHBLNtrYsZ6brYo$tcwN&AfqPjY;38M0vcW zXIxl?q$&7i0kaRY!CPtB%Fh)n8ScfL;$F~^T~nOwg&iBV(;*%go2YzPZAYYvGE3mngq42Kz@WAPiYcQP~pzx%hT$p{pU9R=$%fxGx}N{vQO^GtTu8PoyZR=Br9A` ztq{GBgh3F+OrVI5&7SK^RWTO-{pnE>$ug@&ty2ea58y9BpaFZrMvDzrI_c?P4tqahV1PQ zY13Ncg!MaF-g22l=7VtLvBR>cmI#cHeX97J1?*?fp^Ny7I#$_o)|YeW3a+zj#&%n4 zIJFwKTaDNm6jV6?^UY}g_bkJ!<{1!x)cmDf+nOwdij1#6Oe6yvQ-*-A-%dA8z{`ok z(K(vrJmHJ74K7e(Z1`}lnYHu2q~4R|96Bpc#h$xRTb;-xQ+jq(KBO9u$7TzkW!@N9 zYiYg0lN2n|-DXT@xjcI>%j@c%jqsV~9`qaXEIfC!=|}NL65H_wQSu2*jyW~DUG~fI z#hNkc5R3j4Uf6W=cfWm2k@Vj+Q1*z1FFG+~=h7s#KBNt*;4w@RQZJT)=1=1yN`ZB` zI;kfTl5@9h@#mR;H-s#OciXLOw$gA7Z{)Iueq%=rPBm(7l}Nwk#j@xOt>=J8wSBK3 zQuWUEvR&-Lir8xMYGClQO!gF^m5w01_xGya8(lqsI#k5$ub0G57ZTP!YY?+XTzx1=TS+qFoaP3Q>?kiZ_n%j+4>k86 zR;(X%&Ks7`I-YIoGW5Mms}xHa&6uil)S(@4)+NJp2RmTeW$Vsz>N>_waOTy6#4xkbyKP^tA0{LZw}O!EE7E}LrhdxCRHhMUB>IY%W316YrLdM z(Pex(#WqcU8s|pyA$7R`MRATj(V1Adj>84g>_H^-F-ol`09j)b9P!~a`~h>II5w?9 zqB8GtEgS&=jVxxrW42R^fQ~;!rSgkwUlEZhcbPFR-j6^qR{*%>LUZAvlJ|-QBE$?e z{>lhbW47aHD?M5E3vG+rAo}WY6Nn-7cXW>}ZUY?x92w}@K_~n?CFYQT6yAvS3g4Cm zfG}Y)pX^4fyDDGrE{(hOFzAFsJN>YL=%-?E97JHb;4)Z|>8*o|&48o1uH&S8hrrq@ zcJ>zFh_-^=a3lo=mU9>h&qU~F#W57eG$d@%p*raE?5HEC1J4%gQ;=}nP}$Mh#_3w`Bc~c#T(F7EVuefD zhda~o`sa@!*Xoz*UoaqWFb*4%J9hfDmU{-zYm@6pB*g>Uowz)q|^S{7SgFtEpq=ZT>?a587~g*#j$H&n+5jW`X#)3AE-sBhGxY$(^c3^*Z4Fayh zJc|>v-BW(gl4ZWLaMM9o z(mGH_u+~7=@q}BIz@p!f$u!+E;fU>l!NNobzIhKbb%1p!fp?@Skp$8Cm&JF|b?7f+ z++@ezeY*Ie&K=}i^N&Jq1WQWaOxC&QPLf~pW1D87jj~O{dP}I&o|eU?KPK?>oHXb% zwkrLiQB^gdUz3e-O>4kFgJPY$=+T;?gPdI(cF>v3a=kGT3(SO zX+?MEuuF4P$jK2%U))iF66x4`LPoeH@6I~M`R$_$qDf|=?({QhFwzJTY43>G-Lw5hrr5C2Z##fyI57*!$hTvG z<{=S9yl5=gsNxTqJDMXtUkqIcV1q- z@B~ySrLjA1yu_cgb?w}X?bl5$t(&<4N7*7_2e_48500M=j>C8->mF)?pQk42_)L7j z6+657GD(zUiJk?3g8t3ubzO^y)(gOiJ^|sLzph3KQ(pjF&hgTBUV`#*Q!*?qgt8*EY^NpuVtF zMSzCEqKE$#zO!FZF>cU&Qp7K1m zIm2F?N(iTVw`+aGK<53Yq3C4;JZ=s{f8aH?QvlZc4w8{2W!Wt;p zWa~^tb2?q?cxdJBHpP}E^822?zEi<835rJr7xe@{Wz9~oi8L^^0zUB$#EH-=j$+%LR3nm9= z($%Ogmz~4dx2}E8_(`qSVeX`8GuGpeBMxe%#b;ZR72Iz>IY006K#H1C&gukv@2TII z-&aj8D4E^%H|MZ8BgeR9V{sj+kdo zZ1cM8B5CHt`)RVDB*$}she|BI!&{TilcK>N zsI?bX1}W3sQrMmtzoa5t{zeyClSQl&Sf0~3PH@saFR33(^!!C2zL0i!9!}ET-61ad zOBk91$6~hjV^3^3`X}c-d@Z-w!Gh!$ zw0co@T?mCf#f^?H89*(VX`T97;wnxvH;R!wE8C0rFWj#1X?V#>7WX`KAuitn>)hS5 z61V=zgy{Hz{lwz6e5Hzi((gPd+lJB_$33S+F6CsaSCgke~o zW18RUu;T8EbxM2#h~jGlUtm`)^pV-g1$-7&))5Dj1J^-)x^#)F5Sx&?vxiu`+c;iU z9{7`IY|*B&<={JpWh3XZX&#dp?^7- z=8ydtw7xf;N9qryu7vU{-fZ+gyg%Le5BN454ZxkKcg2d7_`x_3jxeO$HHCgLc4$j{ z2sPG2UQQV9H|8rk?`BB6ZJ2Q)^pggP)! z0kIRh>ODJmNGt$P6r|dysr3Qe-C9cg2?4U1Y_2fK@_nX_GT8F{3~w|30M2bsyNAWm z;y7_Zl0U*%IAa|yYDYP}=KSZ;I&y*L9aGWRQ**j6zXtYv(ib{^AZzJWe_KzNtktx)laUAfny;tnzS4)(?+9cj(CDu3U>w+<uG+l%+!RO}g4>?|wZIw3;{=4s1vZ&~kvV>SgCIno}Gl}On zZ?JLs$l(mc$n5B%wl{=C_P`8IOm4V_v2t*ztFV~UEIDN1ZbuE*&BlA83by{Om zJSKIi3Dl59;)flHfnLY>0b80A?40Fy>~luw`(h;b zCuOe_(zlig4r~W(v3IOI@q1~9%*u(Itm_;AIf-9i^PH)?*~1QM9X;sk*~MI*s60EY z+5GqnMs#O=Qf8x<|HKLjg1G%I>v=97y9aK7fYn2#!{Kv4L)%r&dt*$|WXyVrvHi^7 zfAwblP6alk&9m@=DP~YD2LFettSIE^r)w zT=mQh^_{_F1&cm+`@fXDnHDS&m)Y7WSE!7)1K8M8`J+<>)>1(+cO=`9c!k~>hJ>Yk%Hlr)h6&X+Dvz~tLAr)Di$sEV^r{gS0##yw<8?Q%c_>T zC&`9Z*$*0Fr3R7`RlrwybAQPGg~c4J=sc$!cD*C*ubiH)_OP79TfPDDhO<)G*9RuJ zUHOU_U#@}ET!{g z)QgG_r3hMW5!@k7B0$ZTkVUa%F+ylzDIFbiaFWV* zI##%iY6l#N<{HCr_06C##gRZUO;UNTlDCYWljLcRa_&U=xx(qM<;K^|(!ZE%5z7R* z)`L1ph}uz-q=PUD8b>?g7qWBfD#SmZ>hK&@%c2xBr{(^(XPLYvxKDgMT4_*@HFfE{a z_FR^2`CPAGc2=COI|+=O1hH=Oou(3MI^M7NO77shDl=T?QmZ8-iF`4QIC z7dlttWjl?pUt-$~Rqio=@SX6`p-=Rv>8xxC$+ac3FiCvbOqp<4<6+Kh*B#;DB17%; zb*h6L8I&Ho(2>Z=%(e9PzgPl$uoT_zpVF9=CaEcMBS|~QndwNL7pZTn=O+)Gn(>eU z&&qQ=dR%=a<=6;9!cs~Rqa#!IU$s!lhhQbWbp)1xxQ2dL_O6T!v9gstf6PjF^l6I@ zzkki6W_rM2noOHO;#gNkoR`(@K+8maPFgBweSzXHAiO2bHrdwqqd4$ z0#mkS2T_D|RRh^##{+y+ypss%J^(Z437IVY))Wy;`S~)X6TCkK>?yu1zPGnOMtRe> z-$}<^l;wVg6*K`Un;-uyv*7feFKEm|tVj;l!EA~QyDOqQY@ePvY@#DC5;~s=4WcdR ziZl7HAXW9Umv1pa`(*ErN|vH}v%Sl?S@L{-YjkJJrOzHR`_sL`&m5PKRu}|AI@+4K zyrgkt96Y=<^#3aS!14ASF2g-~qQ=GkVD?Qp`oep1`~%K=1sKeIX2=u`Ep5T+i39yY z=-b=U`kqyXc%l2ouHSpyOCK;Cy!Cu3M#8%;uU#D@w&>EkZp7)lY1_bt5~B9?Hxt2` zMX$g1BcVBN^2c<115S(z`|dpAy-p`JS>O#mMNFk1G%bbtPan(XEvZ+B#r`xgqFMZr zk5;!7N_Xz}^EXet4>o?}gLh74KW}A@pX*SHN1NC_LUusHmkd?FCICo3C4P zQ6(*iGz(I0liLK*&N3l)%>q}W;fsWYj}(rjZ)s{D$BVCo1jo%x2pELC+6Ed5sB1CR zMN4hrE89=*mBoK->BTp;B3-Y-or^1Z4A=wX(Ro`W_%64 zdJ(~MHtxq)I?q;5yH~>065Sg#p;lE*`|(Xvw$+Kk(`-p@iV!i$F%sc9^x;(4VB*on zq?e@eh}_bzE9~fV{0*Gc5*WkO*T$C!)#veY0r}PHO57bE`Q4o1Z!!83qos*O<>h*s zx^U-pket;g?ATmRoUvLS6IEWdzTXt>EjpdgaKia~=W~FQqV$?Ii|arqU4BfckaI%? zJIZE2x2_o0BDJN;lX51_-A=4cThq?niG=@fc-$y_ICj1^+S0>>J0885An(dcjhFo*!%8n*dim(R|E zsK6sNpisaZIm}ztFo}XOMTmE4F>fwau6~SyYn2a%kfiK(G$X2Z*ad%89geXpL#g_) z;rv71wSQ^Qlgte5M@IrGVfxfD+nzvEiV)jCTL4^@>GnEw`;tBSX7qLAvu{fffvF2f z@aqv~rs5)p6`jS7CGP9-{-}6XE2EwXE&chT;23Y_jc`$3BE))4N+|*L4@bDp&yYTGiO73D~%L=h5h9S$O%3?a+e#4;V? z!}D|S9^~L+OHuQ7yw^y-^y%o~8`j5laYx&g=Q_^K7k>UVr$4{F`11s9M>!)i6sukM z9-K3_k{jx$(et)VvP@l-+>Z3^I%HwCkZpD7+7${@qPft0E9TAp(C1ulb)72(maopm z(i`5Pu5NjGV_w#{-Vw3Rek%9xea)%^=5uvY{@mzuy%n4ZGhOS`l8p{@23J#a2=nq^ z?WQB6UQPbb(DzkAY-3kA=Kf;ZS2$Vb0(%s%A0^QHNfQO!FEOvKRN;zv&6g|lkE(HVnIXTe zIzi+%eZOlsAWIJT+_Mcae1&E#>$bKf@;x(Lab}Zq%P+2 zbAiwFAbQy&L~duC9??W)86@g4JkcakWJnrh$3D=shNU-&lJb}$*BjO3JPFqXNX%eY zAUWY~qLgpck|gCxJ#RbVWe})|)?n4fn$Ls`SD`XoMx$EzgY$D0wC8h;jK!VtXYzTd z*)=(Ibxrb$$mdg9s!%i3SVI$05Z$Is3yso8q*K^3J1KIT!)j{{J>$A*=?-1noCFIw z3rF8-hqOOee0j?4F(Prmm)CD7V2&%ROLqufo^P8=&+W0Lo1a&=@t6XhI9uS$=P@q> zPu7PSd3}dM(|L0FvhtQcTP;S5+PGhfJ6$Zp^0L$D&L>3)7NV8yfj%l>xe^~n!k#-P zKn~Z2U0U?5wo^F>ae{Aec&XZX0w1>I2e7ovx+{i|f9&LLTf)HXN+qzr=Y|5N{b*kX zJkg!JRm*#rwR7x%l$cvJ@%AhzSUET2GcELpT~{j~`_*`V>?BEw9jRqwZnKDTtX@|j zw^0@COy=@#MR@krmGnY!sP6z4RPMqdY}l$fwVMC{e%L}N+0P=^?}{d8d%yG2#H8!n zpTk$33r1JYNFD&MSk1FG-*ibGtc$R!_WGoY_P$@qGjlisFN5LahXWG~31rhc;nKKh z^vdVclDW?%pUF-{!+xTbWe@ z0&u4hB~2&vCxW`?d|Wu~;EWtKO4+S2;{w*JMO$%_JGSWxBI5Ayab)H5YDkWvqt4lDsUl&k8rB+=7P-0Sb<$@N-Jt|~A=--=&YotctZM>&j!os(=K1I%I$*Ez(8g=O8h}ThnTU!>_~$Z`gv~)D|zC zF#lLzZ~(UZ$C(cP7U~XP(cfDaH}Z35b_d)5#ix9_^ULi_frt4Y2LI{K&zoA@oKc*x zSbsIf6@YRFqkoCDeIgL=Pr_)L*s7+BFFBp1vG^ib70rp2_LBHWl{7#CJ?&bS*Nca_ zCjyq;9!P0TGSffB#(f6jkaXeSv0g;2NAw*gnnBcblnLZ#yBy_I8f$UL>83IvZr+Xj z7OUc9zG;hVxwO#QdCBDr*3RNeeLIP@H?!H3?ffOR3nrp+93^dWVri8i9eYE6;+=G5 zVTTFOIx3*Pixx=JJ=Xt>OiSa6iG5gS4yjG{vbv2pnWCD9oTiG$2U`8AO`Aiwg*lr9!{t z_MiV^6S~dv#XAlbZdGqX{3~9Fw9SZ4@7Ef@^2>dc_wBIhneFH@*yS z~vA;O?0F|4d5|yq19D(=VBzRvB~P2Yo9auYst^=!*PLT`faHu1GNPL z%mx#?P><^q-PGm4rexAvY?0qaa3&7Je^e{}9Q-(5uOMA53vt+PX0ah;DdbWHbNaGn z>e{z5{1j*;3o!~Q#J?6ZCHH&I7$@fL?5hU%JtIyR{CQ{={wk*NUZa1}i4`uZy6;zG zw$^UG07m8T;s@|y8$YB8R2J-X_zzDC`1t8)00LR{zG<%VO<>z2a^viqrn0 zv)P-5Eaw8lqiTOse~vSW6qZV}Ss*`q9bzVXvc|@nq4{4IpR7+>R~^0X*Bvv-T*9u+ zw91(=|LUA+$c{08ufVw!{092sDsCdrXW#V)~jYZ`lbvW#Z`z89NBqb`2%8H+uHAv7M?AuEJ%x=f<;TQ){MMshXG8* ze^ta&K&Wz$j;G94)En6V-Nm+k#hFdjGxq%0-sP<+iLx3-Sxl-Sd{xyx zkehIT^!$TFe&PI0n8Oka@khU0!!K8cw@fz}XqUcEX4o+g?kUqx7BvofbUBbW!#d1$ zmNW*OZLoOw2|fFdUnM`{MG%&zy~0!;GYP>B&3G!o=-Iwm=;H7sdb*tYie=BZ#X&#{ z2e7Y04|%1W+{`XB^Lj^9x4y^c?OvWFp%fh}?xX1E)U)#M?n3ssM-T`G)6xMb?p>!q z=Y~O0$!79%Yh~Q3zgOb5Zo1>4&`7EZJ&GCgv>Tkxjcj6@%UfDKXS~;AQd}){`++VV8+<3?BwsyBJz=XE&p*X4U7H4E z=5!|JrMEypN4N1V9y@l(a7Wb53MT%dQQ6<^tpT`nysmd!LkN&N8cU~^jjmYMn=2zg zY?hLl8K8rUUmr5_Nhu%QV~0<#1r+XH{qof-r^@2w*{QDDAOa7|^9n;cyCep2Z(0^9 zrf%k3m`s2Rm^jsqz4ZJbM(y%OR2(pKt7@JpBxE5TNm#@5x^&v+|$*UW4MVG@_jxb@Q8W z%1%&w!}qS$95_^wOMd(Mm|U#LYRtI&V~bC{udL8R-gFota>b!)+BUC*O7cPXvcub4 z5v(E6zOC^x+?-82r-ga&w0JJW&ca`bw|72n7`16Ve89QrZLBOh(Y>d2o!s>yV=GES zF5}92H|PD{<~dZ8^;*FK!0`E~7aPLNf58RE;9bPK^82A@5%1z)azUak3Jdo3R~&m> z9vW>&LM10ac?)7@R1x9gpdQ#2-4WnnT_YXU`@Gt%4M^$2hHyF#?oW-<+7w>rtAo2? z`1`Gu7D5*cfi!!)np%a}Ct;(Cv4Ez#Q(NOgi{ta@co&9T3P+tRXRjk6{Z!qdEh2&T zg<4vTKUeAaf_=2g7NDv&-GnuVok)iUQd5=!3G0r=tv!as@|V?qy8-KF22rc3Mq#EO zOS2zQRSzz3lpfZ10V1Z>H%2dIcVAQT+g;`4pG^Y0zbKVm{1&zx=ZxOC&3+kfIVT42 z7XFkYic`!lQlZXEa;dK}p;e_(=-v8eOxfLZKVIjvlw}F5;Vj7pY+0P>0JmJ2KngcV&1eR#+4RgY)sS5l4Tr z;IA2R_D5^4P0Y!N0_Biz#O(U%ef~6{H%6=j^EA+!Q<@^g^i=GU0fs{z!((za$8B?| zvYR_bWBUK%>n(%gXuh!T0KqK;xN(QzzPKe2oZ#;6wk)m*8VHNK!$NS^#e%yN+}#}( zC(r)Bz4bm-&zJ71sgIRrHj=lgxeas%5i@nZC0D_Pi5A!6Vb}F)~!;uXRR0!sfC*m-g83>_a_@d zAy9H>nd=f`6g(F+=jUrBoxNet+^X1V11w?HY+x+}1W`-p^TJhrmQ(!6DmB=7xq4Fv zB}Htm7nSeKLSY!(rg-0~$FVbwj_MHO0bFfr4Oq?h;jz`MDipfPS&rfANv*~}?61SfIUwrOX?mi=$|Gqsj8yE_YDq%`g# zCKYzK&5;x=Lvk24zh2LX*`j8Jeoj~+s9JTor4)89Wysw3-{@+n`iz_jy5)~FMJvR+ zmf(&Fen^NN;q@(DWq3@d-}7u$!${D}p=@#qXJBux=|nAJV^?95P1M)yi!|%Q3qz(o74CZaeP`ZQ9{@ ziH@>e5>3(hFN?wI6pVAw{nAKvR3hNuK)?!ys)%AlB;RJ<5N&uTdeT&?D;ta zMkvJ{ruXKGFFr%Vw~ZLRoQtf$-{dw?L>qibFkX^xg!Car-~K!``j~Td#DY#uUe%UZ zt38nBF zW~eiWaRPUF<9B+p`DT4^s}K9Kg+O0$1`d_Ax^UcK))D;n8Q#P;#FinWW^eN1i+bOR zAl$$s795cd4x|1p8^iLcn);9(*d4z}()YoZh}|_TOZrKke=tLDFn$C(e~!d8(E6b_ z9ka7mcM_PiK@jybPc3k^!Pw)O?L%F9b!)xEW!ufK-RUkBy@GpW2li}0Q=MpCIuwGz0-Wt^ zx@jk02Ic@-kN7Kp{bCbx7&M?zwj*VjG~{Z7(%b3{6NYD(PNikus5b-Y3oAFohMm4d zDrFg}0P}N?Wk8SH!5#ToNoKz|Etd*iqcHbAh6WE@Qu+n8NK1-KfTR<*%dAhwkadLH zsG1{Sbh5fJH*@TcJ*KBBX-3&t#lV45TZO z<5+YiNQRMAT@?A(ZLf4jdCD#Ya~a`elqgIuPyec7=w@c9jB+R_ozUl0tfDX=+#SQl z2Yc%y1AAkuXzK5QX{}IOqvHL-IFn53%Rmm0)PWtS=#H=>Zpo-PbJ;$`b=S zcASW{4xv@PgAtah%|S90 z{r)?-E?X8LC{C;G1a=`>&2UE5eV!f~3@wFNd|l zG0>o(&gM#hl(;Tf#_kl2&sFQ4-(HROa!K!Ax-y^;&nS?14m9J>5HQxN20I&$K0fMw zZsFlxW(QEKbKH-dWkW^o?FYo@1}8As#kMpmcwOh4~=)B1AC zyAs%kMXG|CdIar~ppxN^6sc`tTz`t|JG_??|I{oU1cqmO0z;~a zTGxHrS1*!0m|=@K1nPpUfJ)CD@At*b_K=f0}|8PNlc82LbahNe^fKaO+NIjH`a00 zox_UzCI#S<7hyoP4dWnKh7oDdyXDy(+WtEB~O9KdHrL@wu~#7S-CCv2TF8 z*Ubp1U3Nw;&{T|Cch0nAd*KF zoZogT+hb#tjQ$O>`^$=#hj36c%QUIfw}3upK_;tE-S;jZ#F4xmTK9E4mle0p{zC{{ z7o5v%vX5{91r!R#vn<8}liaBUFG1vc|0H``Kdv@cj1`ILknjW7?cCWvCKww! zx$fxsG=EoT;+TwA0#6?g*WWr%`}9vg5Bj7xgXu2^KdzOp9$CMlmN;-#UxQtI>>T|^ zWh}N0j7hVnb#%t*ZL&1A*&kA6Am3d4NBLt~&s8bi+#Y}M^uq0QCA`6l$tRam^RpGA zt>Z=FnYGKGT>vvuYn);+FOIru=>2=B&gl*{-J$z0=T$kqIG5%U-UXzOjb0x*{#h5PJO}r?`PH8NB0a%& zTpLiMp01N{U;v)hi9_XcYx{1aHFtD_|7Ky>scd2^^}d> zBA4iIww~7d7RhB&oJtPEZRmTb@G^SUsk; zio=qH4;{_^Y*FJ#yoQGe4obUOSvM|;@+REsWc(h8S99`wP@wrlL&rTfas;a*b{a{Z zJ+sQMzOTq~cA+U-og6uuFq{g7wnPOOW>pNvNZ4e#i2$9%>`E#AANy$d&>`M(t+1*4NC`F_J%S1&y*?ur9j3>BJk^vnfHn=I%Zqdy|k!E zcNjgdG8Bp_kI?v9V~>8B)yBT|*DM~(fdR>I>?NLZy82{O*1kPjrF1(#2^3F@Cp)~( zM~@Vdvc5WrD83jO=3`et9sz)_v`4lgqB(qak}>(K&NX39)l}UpVfHZ7Wb<@f`S=yC+(!1Tq7R9LRMV{?Hu7A_OQzZnppWXdjzP?Gl z3m@G`P5LQN%Ic`qE?-?&%=f{v&0xKpSv|r{%{@!qJNlSrcI&{xk`@KTD*{rgfL}Hq z$~JrNWaLPX`cV2zrr^TxMlqoC{O5BP_+(NYba#SB1g!{sHS)cnSwQX?S!*~fny&rR zCO@qR87+Y5*XOAP>_%iL5_Ltf4zx~}w5&I>Nc{nr7+Wiwt;i*{omNd8u)UT}z-Mvk zZI65IIz1V1#iQXe+n=KQCg5L?Yw9}afyPJ>PVZo~k&!roN)JpcoMUk&OJc|FGP0p8 z%dEe5jifA)QtWD(Iv z*KP;lz}2^)Q;`8Dnuv$aUBN^9#EA<<1bQe8=sL*?I@|dxKCz~3*?Byu{Yv}iLT)lQ zX+yKj1^<+a&{TfM&#t!t2^L;?a9b>W6S5(fVdOD(xaN9>~Mqhxt}f6b~22I zp#FPk$K({4RTh_@w21tG(9Cty%nQ5=evl zJd2ps?ncCC<29vJ;U7i$OhI8XbC>FM3nO_O_WB#H)P7Ndo$kG%hN^$r4)J9A7a%*n z*aM4McKvxz(Fhs45V4ny+wl4u&F*+n5`yl%t|sF9)(OF3+4Pb9ri=Wd>PBw#bGr&t zT#50;fXF{nu>)fJ9TWrEn9kc*KMAH4+3@hIf)N|>(-ug`q>3$gM{_*0nXcC5PKav+ z#xAKePcupUUHdghf{LP~vaVK`M$`nd(3|g+1Dbu!F1>DKF<)9qoONd~~8Rv@=m@W|2Hv&UWB>=^SoQr>=={yFy=S;~<>#KeQijKF9lZ2rc_~i-p zA@enQMTq=!WsY$oPe&uX`0ol9HkjG59!MteT|fl&zTshfU!8oS^?J=_t}_hn0rd-H zRBh_bbtbtN)Ztg+47i5dn=rv+pkY<2dj#RFA|J;2OoB3Vue$58dU_@Izt%dP@ocJK z_}Ys3K8Iuw)<=|eaQLoR<7ycg7mT@ZN>UH`^BRJuSDTYoI<37vwGx9J0tK<8GDYNf zd&9=qg`#m_CK9*#?icvd+S+&k69#`9FLE1l=Of=4+o{h6ONvbeKYmy7`pG-xXLsXQ z;lhHZADo+?a{oS1a<)UR(t)wxc6fIvGY4#B8BM?!*t(0Pxg_ID0CACoJ9~f6!i%|w zQ`?2|t0lDwot=6*_v(RzuZ&dS$w*Q09DLrxdLQMn0GZud`5cCCCs8(gGUu_LcDcye z3(=y^C+b!EZ70zkp7TBQVE_5y&z&v2;{%hjzzLGQhDieBc#^%qgU~s8@z%=xf4e#V+^|SE> zNb&sqMYxt%ddrq?>oOPOJQ@?$CMkRuuj?|OLo8pGgr?uzrx<8b)!JE8Kzht_6M{e4 z&;#8I0*ug~n;yQPhDJ88*(SyC{pY-(=ciIAg7S~eR|~`(;ALok*6L0Sdrb8J=TAs_ zxwW_H!q3nMN!VAszx^2NCy1(eNR#B~>M9~#6oe`>z$PMUBUsl%({sb=`@YnOYDIqc zU-H{aP)iGs#{Zq5{r{=j^Z!isA^b^vz+tiJq=jT6RCbe%*uN#5C?iI{? z{wO~#T>QIQ`G*Ef33|AD{`Y!T{l{Rp$Z?nEHEcNdPt_i;Tbal==HTCZDyLSb01u1| zO7U+~NkIvKJ$%W@K|!^t=-XkdCX1{f0zl$^2Jjp}qe!Z+lRdcXAZ({TluKYPv0oPa zX+VYjF|FKFUPD4`V0)O0LEmqDEm1|`HBnYE$6x(7bVl&r0WCukbmw{=whR5O5-H@% zqtqC1j5coM2f?IkdgV;84D7*LvVz{L43|PrqaW;O$k}wDMj^U6d{EHzbQ+Y3w!*xP zCq^dsAf9N)Qkayz=+Mb72B5<}z9TgFm~Q16dyqB2IKF#kJ8?GaHk^Y7LeIr3@Z&jz z1@u8^a6wfK1`w+Y(_NXFQlFNcso}JGXav_R<*SrRIJxY& z7Ik-gM>bQkch&ar2NhjAz=G23Uh%9pTE?Y-ycDQWwVj1%zEBh}`*Cf>B8k;py zS@(Q=1*RZtaeEY)a+nl&I6&yK;Xp4$^e@p6$RGYdk14cZD1Iv+c`g_>KerF{yn%IB6tU1qu*c>C56CsO{UD z{ViqR=|-i@vDMM*w<7Mw<@s4m2MMO`etoA3RbjoFeqqn}P9LFr3-Y4P1fIX7f8O=C ze#2W59$hY$mOTeBiZ`@FhfbIP2r z{}I?Vce)??kb?+RMj~?tr;efSKyP(UdvYsj_XN|J^D3LUH0gIoi@Kx6wM)a6_l1Cta%6UB^V28Eo3P3k&$UM*RlxM$@ zSdC@*U51$3yKyjkMud#<_GiUZz%}>Jnwu3GN)HEGBX^#6I4Q;c-nM$d`*YzJ71?RD zmSp1lQli(q-TeMe!}uT;_gM|`ef2=p#mf_|+QA>#xi_^%*KZ6@^zmmq&dkvdq5+;L z>|(K}Q8t0qK>P(4NPbangkA=!P*<;7dGj^-=GM@G zX)DIwUYhgT1`pnLjVbe3M(;-V7H$2f==i$eA!F&^fUnWk7tQS6#?k>!bEYQ*bv&3yS0tlJLK zrCsIe?HDTT33PMp7a?Q7E{L4lBcTl0Yx5cKxgnM#9mdUyh8K0nIgpH<9D=l5(`2XL zuHBF;JWlOsEJUpBryJ)|+cpe--_PjY_$BXF72tbPR`|ioSGEJXiA`MT>B{cQ4pBio29*aeGstaPc+2F*d2AO=dU=+%$33b$^FMhn;fgv0+-;jitYm$>@96)U`-wZI zAq!t`Q>fLZf}tXy#l>aR@<-}sRmYz4#_yY)Px;i{4pVLARNZzTjzs-rAzB|N-BcHM z0K(=*{;frAp*diB+a$E3C2DSducOOPV7#@n0y=&NCr*{%vp(;+f*>x$@H@3%?nlSU zc^%}1P)^Rl8fO7N%9PZde6?jU$$*yx(J{GZM7HmHd&KjVmJ3qsjFUz)U3Z%}^?L7m zm%n&ZO2({ZVrWZd-DD|}_M=aC=^6!ojP$v*u((OgUmHBVPLzlK+#!~=TfHSNwvxv> zabwK-^}l|==`GT1?f2iOGeFJ3@tr4W+CkD=xaR%lML)HbkI;YNcZiNS7&k-3=+$2W z^;(^Va_QdOne+edUly1AF&<^Z6$ubFS40Ji{)kt$!@exAnSG`>N{R`d^ybfQj{p1P z!zg#^>Xlq>7>n&+!?69bvt8L3jZa3mi82z(w>URvqrZv7__xztO!zwvyNE42116T? zwTFH#xf>!yOg+Nj$&KF>8M)+1_e3a0f7t$t>l9B#nzkn43MHIrmQEXz)TMcK;@ImM z(vjOp|HY=|Nw72{Z&{+;R4G9E$RCf#y(5(D%c;N+#`4W&L1P^LDe=;P5eetMHCLASo!=af;S1DTuZd+7KjJt@zA1Vg*?_sEK2* zb96TlXMm}gVuiEkCX9s5*6bnu`+9QVz(i4dsq$5x?iIs5aEQ$@2kVC|adc13qlq2Z->(}lGJi%+1+&M|A-(7O={Wi7^@@`1i8gVZ{EM(V6vIpCV z@4+y$_k5{!l}F%_Z=#bkGK~GdQLjY*eVh3#e$Hfh#&HqKzNcV6eyzO7n4I*Y@)4ju zLdaO*dB3mOt_sQ#Rm4T>iv2hQ^Ivd%ga_2- zktkk2*%h`Shyts2V2A9a~8gy;ES zI&(CKjbHCHEe^!npDj3TT_!#|847(F_9oNuB&Zp~%-)I)+378QMfX$0gb?yYqx^TE zGUDM~jH(~x)GJ1!dc^4f&xRdq=Cg?6LJh@FY%3yJFS8n7G~?)NFgRZ49>b#XywnEo-YDVPFmHRJqgtY5g!g7n7=q`;eQ#s@QN9Dlw_`$F z&0|p|Sl$bN-~h#)8H?F)NxF`dKb}i#VY`uG%)x{~Drni^#q9Q-yM5p4W)<&UaQeEKj>x z0-E1KyOM~dOSgnxMKzqwKiP3es;MxBVa+FG7@s`)GVegv<4)jARbT^|n-kTaWtX8w zI{1hKflR*O*uzLVY$8H3Ito)Atjat*Iw_v1k_J{x-cXYVs{M?hT@IZ_o7NvbC3Lt$ zyUrHXGn+A2v3gvEm)%@euDEvbkyCz>-0GHE`*)qbQ~aJ1PMs`ve_>m1L%LSeVdzAEOW3t+(^@pvllUh#0(vK0N*6XTEL{qJ}U#9hlc@-hNE6n8#xH~) z5)}^0jYPuVqW@01ir?^H7RYQvvnJbT_-Zb0{COHEBy3pwgA{eT z%_1SY22uo5?~!#+n(t)D;A`=@XEIw8ig~<6hlc4A`m6StowICHe;<}k@6BC@GlJOF zC}2=Z6Hs$CIfY>`K3ConGvX*m)7y{M4}4m=spJ;enGKzJ^>+~+RoPsi+Z{_j_YW`- zKjqu@4|v{V-1TL__XP3mWf2wN>ah-;_^tKI(Xf3W+gqm+TCHv~<@dQ;B9&kLxp6k4KozwHc zwa=YQwv==$Ds7pkB5eO7z}~1U??BKfXXT8tHg}GZVNH){V&L)1^m@kUl$d|{{-(#c z;IZB^LDKPmOmvsa4xr>v6rD}c>b41eDk3&R3T~0l>FX(nC`_Zd7&a-XuGSApCig*1 zc@3`@Ml$GSS(LwGZW*je+(Vrt9z5P=n1*PVV312sF#wNlTHHfW3&jgJz*Bm7O}RPL2r*31)WJdAinDsflvwoe;n zPUOYXG$_9AeyU^~!pI*J^7VMS_EyVilV-Zsk2)w{)u~qweFlk)ksu#T<^w+bVU*q+ z<1XhzhV*66iQHrL`-_;JyE)Ie)_`k~6eru-;^9@+yH?AE4H-J59F}@K#yz7U8%mO#?NWKg&Fj z*=p)NxC%oC#0P|Gno@$YQx(m~KJ8!I#c@;&sA+Y}i97bM7+|=ZkEFvR8j&75zGrmd zhNd`{%NGdd=SDb?iW={3;MtjE#zpXs@U~q88$4L$Gi}Jwx)RpnPxAPo{`vvuvy*|_ z0`ddv!cAQ>X3p}+UArSUBuB^MrpuT}%Cg_MUDDD*e8a_#6roGE|4#kKyt3{P6^hex%oL?2#J@#q&e7U@5#V66migj1(qk@e` z$Kcak&YceuF9pd&U{T33&)2^J2)xZu%a`=d_l*Mg4kMQf>Z>+L%{E11yEm)TvwDT3{qr4 z?01Pd_eLu+mPW2uLeHI$V;#k{xp$@(cY5%-tA+6M^m+$K4x{Iwosh+XKX&qGX=^!c zBd0d7O;f)5Vfd)Mh#PJeG+g^UL$em=T;|tCdK6BVEuc3($2(1bZ?Wv^s}KAW>=dqo zJ)TOFP5Hz{s1i<5Y}40_+iVwcJ7tU$>C$AWk^fs|xy-t0ddRJVOaA?MxjfH^P-+9? z5r~k4>v`%>vsWG-fXS`IjvC^y={p^WjgFlROuq|T zKy+utjDx;gjE>HNku|ADZXxtF-r^)>of_d*8mq;qz%pnw_e~3^AV1INm&Tpms>MiT zIMl<7<0*&l3j|AUH_!9_Hwb5or-3}fvzw5dd)A@G9${&eg~p`u`>9njakXf#6M1P% zVgcRY^i`2aF_295i{7!;&d`5m%$q*$7iGOQkqI&+@TrQ2$0_HEe9Rxd_~4AcN-X!= zHCwD8PU2laBXV1c$HmJ0xSC4hx=n~*f1eFbaqom@-Y!-@NyCMtccu+rUG^O7xE?$q z@$nUiph`^@;l?gQ#dVo?y^8+@!Eo%Ij(Cy5%7c(n-`lX&co{Eab%w0?YB?|+ zh=$Hn%3AGC?Ud4wz1H%QTBWN27QyC84#`)6cx%{P*OnqC~`GHL9gB7&7zs*!+ zTls2Nm0yK@ga*`n-M*6G@lTa`2;w1)8gje;Pd0lxUYT1yO#AcX*@%M`!Jp#na-(E| z%9pxJ(hV+ZUIA$>(p5rFXz2vP+@c{sTQr3RDe()04{!QQFQ1KeB_(38@ZQA($|~fa zf#jQ0e+60INn!}3GBz%R)s>eYUh&6Zd^cp!q)d2_!plv%`BnJRHuH~(kB;z!!s5k3 zuAdW{_{qR(@w!GvC*xEK!s?X|6D&WN_4QP36X3#t-?yS>TF&G@i!i}LfhLbpO47&} z%)L&E+4a}k;b_U#2Q_SORn=ziZJ!x~X0zQN-)_n%8fxjzDmhi*-<5gm{M-+gk)9qw z=33oPm}k;2sW{w}g7}`~W&AgYdD;jG!$2ZDW8dpTM|l-FdE%ddi{#TrfBR)7N_%ky z7SM-9#LJ<|!uOKN5Of{mGFjeRLC#40^3by;4l9NNd6oN=7&!yW+q{ljLyivx_1C^T z>)B?<;NE6VX=g9v@AYm3-^Ljw%Es|bs|>y8#m9#-iKA(}_A}nS@?9Xhp;S<-0r@kS zZFGi^TZ&vel|K%{?P*}Vu_A2atfw{@(89oc0t!#n+H}_G#Z2X>$1rvG0R&x7J!639_-5)>H{r2?6#jSrjPL%gt#wCdgopmS54c76f*U zz#9tgsHnMd5l`EyERccD>C`IQgwx(O*M#9Chs()W^2K%2gyHv*s>OWfy}3M_7F5C% z7vi&|eEX?H2?j8TBOQ6g`6a*Jv(8@!tmC(UH|dZdL#-Xx!7}K>KD@H#r@~_)Ixwh(D1g6P8QT`k=4pj8<*r>$?NmW{NN1r2W2Id?)-|ny-uN)4pgQ}L= z*z|9{X`0=8o7Y6=>h_H?UiBN4e&bs7C3_YwAI{A5--Q&?z*{Jc0r zs*12n0bb8X0M_z;LcXx3kQuf&MD5OHAdJ%8TLu&xHvzN+(E{#GZ{g|N<61m4Bw~wW zQ|B6RPlg{CE>ql)&XYGRme@S zjxazD?)cQ&2QOZAe!;vPIt0}$BRU;D&yiBTV>w{uyh{_kNMUcv`oV3;2R?1CqVA1# zAaw$;SY)xa{%1r&EA4a^IkDRwYpX9&&nmVkH4!xRLtwzTBC=_k~&P_lK z*z%=QaHVcSvqWsdXKqE1WymwttWHIg(F7=&{aWT$OVJ&RlV=oO&r2AMT?KOHEZ=YI zEoYY`ricC zdKp_8a&K~97c+8{M$B*74Pm9D5dN zFpvHeu?!9!w`W$;!fI*_P<=i6e1=T#2oASO6kfn+IbsMVTQWOKp#F-I-=0Gl%ut$g z=OJWZL`VQTIxogk!~M!YJOB-E2&kE7XN(|b!|@OmRPY5~;0pwEqnMS;c9gLjcWkr8 zS|&1oquO+hKjyCqJ_cdm_I09KD-RW14#ERVqi(#%XYVr-`bdsL2F8kA-D7XIlE>dv z8+GC$RiJT;9>>EKIR80ls7KiONB`c5oE7}+ymlaWxzai*9>!#E!T_FWQFG#jn{!;+ zYD_W=egRVm#YE96iKX5qN{}}D@JzK=bBwR$GdClfZmYB?{LWKb6wnPc=b(`NTr{0% zV+28eDNMA{f}k%4c)0NBtYCGtyyOXB?Y!{^TAC9$tez{6wO#k|N6i%BO)`&@p<3Z< zGeE#HM>AJtmX_67inP%;Mrz7Fr^1X6%QI#H&o7#X+lm+4>8zZ17{3hGVX^ zmTlU0b*8qJ7brUzsGxUzR=5%&yjC2so>XpI+F>Avr7x|R4W!(AQ-lb=Z)$6eprasA z+Z&;PAv>0|bSH~s0S^N*@;xp>`GiTNrwk*O>{jk!eJ`f(MA4Xz);V3*Zj^OSUc~3d zGQjiC+Pn^*5~6tPGO(#nI4jz3`?o&?)L%K`TcdP6j}O%6a~ovMSwuKwUuigTWGH4c zh^PF%Y^nG9n3>laG+90cMSIr##1(y?*`G_?VqGWD>tc`H-Xf&L%LvHJ`q`4#l0Y2q>}yAyCy}YiQjt~WQJ6+cJ91zShLdh6{s z{E#<-8LtOBlZ~bhHjbv;_RNXoCBcosz+LUifGbiaBwCY4aoCX=lX_Z}eV;15^NtlK z1xlZ!zZjrSom}61Zh93LhXu2O1s9gm6RN94dCKl|STi*-S|x?{-OTFV3MI-GBO|xw z%~HMn=ZNe@Gui3C!Kr7H>o|fBhvht8Cs=jAB&g_JO6iIC4@hKu9^#p(Z3%EnMoR%$ zhZFdVw(G&~vKTL?7JU%Oz&!}@F|7^$en&$Lo7LmJF@1Y<$)~H%>0|(l(;)Hi@tv`6#H|AO7;crqeJf0Lo^Gut zA6DA>)8EUJdy^!7uU4Lsvm$Oy{tBgUidRN9A`ta_? zxN6RZd@f|`SERQs!STNHx4j{s0ai7C_&zon$g^vxAaY@UHd0CMq8rIfX^F2S6pexC zZ$^#_#z0)my@jLDl8tD2dcSr8XPBN&3ZAMLu$>DDiFwi;&V z8jxHW7*(~l6&nisFmLcru40~$%gQ2jDzu9&G+U(59l(%F7N75!uA2Ni*J{W!Tm;x` zIP>xu|F!B^W<=lcX)fP4ch&`P^0Tk|^T(;y9E+{kxRKv#0sF5#9vaOhHCVp2u#Ft#xM}-IoSW9pfPB z383P+;VQXP)}ZY1s;y*w&AF%X^+SQ#DRiHR&-_4rl9O~mISt!HpfURhV?lVKx$Yiw z`FZzA8^%4><;iXtDMsWv&G0CCz$vq)L?A_bS;hpnpmZX!vG^~D!(CjD*pU(c;nzL? zB#W2q(Lsmm{QJ(@e0rUUSInNKo5~;QsQ`=<$~i&(-ap)1xFQPOL1waidYP#zs}p2k zPV(}owiX{@hev++D)A%&M+L=3C_<@PKmM4hbU^R_ih)yag zDy#CyRZ-~+pB&c??pF@x(~plq4nd0FFQL?@`_4SR5eCr3dvcF}@I|Y(n4~;(V+h2$jVDS1p zCN!N-K$jS-XQjSOJ-Cue5=gc8SDlisNZO{~Aq)A%+9*Sj834>}_RRzv7M>>edK zK&J{?`MW9qQB5?KC{VML>r97`vtH>PV_WQLYjfL<&Y4!lql}~gDCz@}ZuZyK^yZpRA9bbn&HK^nm4F|0yQ;dZ%pIla~sGDW|B#i&qONxj;MFFmsRK zhgIa@3}MP1=JkfHB^eg|Ax4J4Pvy<8-<^Ng>7v(p6}CI9ZBIy zeb^$HgC|ezP5-LlcabI0>&?4z{&YoM>51@{;>dIXiWMhDr0FLETFiXYe5e(;ExojG z>C@O6>@+a!jNp3)%lt5U(n6hIXCL?*3*rb<|6y~N6ypg!82w!16})ymEEas4Y~vc) zq1V-0@v(YBxr;R@2XjpUVw|ie5My;2l1O_swyeQz87?8!!io_(S}_n|yU_C4AsI;f_2*LZ(HZ5( zHviaw*F|PdJIfk`sMNeiW=oU(m4hz!$%TKKTcdAbNQ+_}t&veb@$RHQXq|pyp`97k zjQyfHH|nXic=2hKX1@)im{$#2?&=J1%ktGZzWDTl=z*fs9!Mt3KN>A%ZIujA_oA+M^AKtBktx_Z_H?`n`Pw_CCD;s2!sqE8 z5|lH(#*&(R)3p7k*~6X85*--@GfeGC$COrpa)bq}=ZDAzCS6bM&%s91F~L|ZXAAJO zAisw4v($PRNtN$MnA*37vv1Lm{I)CJr30JEyN27kSlyV%YfQL~!)zS*#b82Zh4VS1 z2bCn1--7bMWrfo~6n;cjbfd{6#(V06nesg$(oh~|hJcgIL{Z9mNRHx{?}Np_XD(UM z6937w-Yn=8UW^Epa`LB!g z?&JH;@81n@&mGXxy6v^BseGzpOePt(u&rw(VH3-0N=7s7yQEcd9lf|wrJ$0v&Jn8t zR&tjM5!0?aQIfhIdcDu~zllD$GEs6Gnr#GRwJ@ip$qIGOOtK=PZDq+bvK|vMA2|LF zFi#5(xR3jsR_u*J*#3RqsQ9@Z$(U3+e6P zcSq5gWAt`TpODIF5B)=^oU4Mr|*i8G3ndW8)RZH zs<&@8bQMuVQY5-}EI8ZINOaL=s}m!bzOLw`bNbPM-&3nZ(T*?W*Ms~qEH#T9S#@+$ zw$n%L1IrkhI)vD=;p}62+Q>-B7tSW!L>Pe^=0<|HWlT&Y@_nGVe@d$SGOM z%ZuQ1+#JU8Rzwm_-42A6qG1zU+OiYJ19%ekQtY8PoYTET^pjm0C@b=FIN-3}uCIz_ zOXFjVWYf+`B$uWWdev5wbF}3IBTH=LYTnuSuD^(4L0n zEC8FMnoJnGc(p&~cj&0#^FAX({`}DwT{*FDI4JM05BT?PQkOmFOa$TZ&dPR?crTiN zYH0<7TCjjRejh<^-tCi7FdqAyjJuB3bJ#?aR{>u}Ipe)Zx6_1zs|rf9wY7Uw?~R4| z@Zbz{H#}kF?i(qYCRkfrq@CM{qCGrL$zVtqn)0m%x@dcP!@C z+P9(6|M+;eoK{uzVj#4VdgDcjI&C?M2s)i=beRW%_s%!ArlS2@Bd&$z@BIYwr9W3y zn{$wjE54@BX^-qj_+^Dz=(RQ}wF|2(8H`jWKyUuSd2Wf>>In^)b{7|E7RGiBR_`IX zjD>;Rf&Nc?KNepHZQxym42<6*GXh`Z>?C5D=gU<4kCjpAk;@`XYoc!nJrp0h4h@hu zyRLXTfgwnZKlF%IQIZ~2$c{e<``v#{1A4b8jY`q&%)#*mtzfGiS@W4$tKf8U2bZK=vwg=EN5M^RvUQ|OSoy8QH;g`D^ac*=04tp6`M_4USV>Vbo`i|&oruJck z(i){OeC$>eMba?CbRzq;Kz9U1y%Ghc(kGP{i#|X)` zUPJB1h!FJhG^R>QbGX3Zl(%p@M^6mBgDmFbnh+x*0I8C)My2WmZ_NAQX1J-w=<(i& zLZesk3Nu{fVI}qE4F6~A$|M^@w}*6r+#IQW4g9|CqMzwdjYZb>irtNcfA`qnylz`) zpX)re$aNL3g0`O@zi@<}jfG$T&|tW3YdjX;J-3++d?#zh1*OYHX%vG;OfK~zlmlnl z3A^=xcs5znRrZ%LSovyhrQri(!uW(hF&ZLw)>m?A)xs`w5U|)*7B5?j zy0I@)P!efz&e#dZ$84U(p39gGqYptq77H#JoM>WSQJTHJ8Be%U?lPD z{FA9Fi3QAwneIzUHzuq=Ta&((3L_?}X1Nm#QNo^S>+y+fEd3(6FW$0ZSXM)Sym$Pm zbvoUN&ra+4acxaf;aCCMAc1g5G#3`9#Y_^a+^40p^jo#57kb3=UP`CqF*iZ`W)9Vh zOr|k7N{7nOAp1)r30xf##>Fpa)n|(ur@uP12k1h3Xk*+5T6$=YPX&?f)(Jc|Lb)<& zoIZsv8JY6ZjQuA{ey6+tV+o7-x3ABg0I8K^7s1wK^*C+;rJ;U68U>o*aknv?r6MGu z$uSTaP$N8D8cX94so`m%D(l6`6DHsFTKw{7$Lh=C_Lq1FaH5FU_e3k!noKtN=9v7W>{(;V7L(Sv>K~l^_)=J~Kf@O=<3y3>pu%9>8RYjzorc5> zt6NW}V+Sk7e-qP6FtHa`2`!vg=;e0B!NtUipSP^#Hz!}fe_6xnn8P@VoE{=xa1Yp% zyHLyt%W4X}j`Hoo0Gy?n4*(4Qfxv6;>xyf;O{7UUbZkxKeN#nEM#nDWMQqCyP3g9X z@L#d!J>lpj$n89=p0LXhjkJ;QvmG#SK!%R)zb(%nE`X80Ek>Jg1i>=D8aex6d$>66 zEgs7I`;|U#BEU+gl@}+D|8zajHgHUE+=cdDA~y(wM~n@C$j-oW9Eq1gk~f8BF^A+W z_jdt3KtfN34E#b~vxD4vAO#Ue-Qy76naXQJ+5kIo+%dn`r_XwkJD_}utj&I$I)w2S z29JZtmx@1(AEb&{W-1EvtLeAICLW~LxwGdOIk|=X2!4%MYEBO>Ikj^ioSUTOLz6s7xWiF-^hT`>1{ zeq?*iYja42;yTa}f^L`*W|I}ZLI_W`*D18o4*!Lia)n>YVYvPLme(Owd1AqJs=4#L z>P>S(@E?k6SNVnhW4M$ME<4aJpPMkH&M7#JlU zlSq||=p8^t>vB;w-00g7dsi@QDTTapD{Al&%D-y0j~pJ}5O;VXApHfr``)#+pikKK zlj`?5XZ-40#|))H{WdDg z9ym716W(w}l>Q$iA~C^DXnb)E@V_$8n0pMTihq7Q-~Hj`rg%{~A(NqUs$~#{3RSzl zu({cvT9eLEfMdD8VnVJX5G}kr1Ann$*-jVc#{3F6MmXO=;67SAP zUVQF$voE0ha)}gH-pNBS+h>z~nU#B!1O!%CX-fxPQoQsgelPbK!)6^o{f)twuK#_N z7s!c6LWZWqTN%W~SLuzU^(gJG1ZO!VM~G+?)RV=@K%8oxCq$@| zW1*2J0+htJV=sLEWb8kl>CGa^dduIha$|Y7+%ns@tAYw@@lp2TjCA_KR+(HGkoR$2 zQ1?03;&xbZLe=d+_)}5}-cof4AADr{9cXbUZ(zgZKk~#;iY?)*X z9%aaiUL{4+X&ImRlg_~Jf8MAj7hbQQe_)m)_H|u+Ge}JIAwBAmCDgVb-I$ZY-Htf= zT4Mc71zl8M**tg4IB{`iOosX^vzD&qqzI)sc~GqwQ@Yg8@y;(;0;SO zrNlbC>VN*qwG~(GAoFk)n26;1Oh^MHBv9Q%DYqw4oVwN?l4#|=DMxOZl_}iU!#mHP zB{*YifoKFL8sZ-RbD(0j>eyPYA7-{OE?3W|a zz8*8KMl_LYY02Uw3?1+&2JpM1aZ%|p&Rm_V5@oAV`2`w6Epi%NNWh29b(Z~1*840+VeDbPc{UV+!*fG;LS`kq^UnhEd?d6JKa z;-oTZ0LEb|1K1lL_HiRjqpiy>hL0(uWla`wt@9Ga$pg59ZhW8pDxb|>N^kMR9d>+x z85s(A4#HxE6eSoX_-N?+q!Ar+54)Xhn__meYt0)ao-%?b8#3iqc0JEAy(@-E>-pg1 zOFQC;tzDZvHko2KUo)u=*{k=alHD|?=s)jdkKG^vTG(3Z6}|K$)K&oQv#FpK^dSze z!~21C8G(_T&MyuZP%>TaKsJcBL;2Bz@EmAW@+mxd^T3%uFtJjOvGE@4KU*RAjpjuN zV*v=S-7j)<2Mw{ue)0jGNDR3qpQX7>v5yU6TXX3pv>%d&LIAhhA?`R7>~C57AyMH{ z;imTc3*2c<*0)-Xt)48v$MwYhq_Mm?)EDH?P zK#fllzwYnkM6pZ_4+>CDkI~Gy$<&T!2>2qFasnI@y6lz)YiB`R9I&x1QUy73u4?I8 zU)d~K)Mc>%4v0rz^-RDf3=rap~a}Pgc`3IEt~x+hC1TrDN}0cT;6# zXzEH=*J`36k+V=LnqR#?1p~_yttAiVr(B!YytCn!E?g;F@0*l%f}Qn5S$Dz-4k)9u z)bR5y>rz6vC@%Q~-k<%0hIK#V{GX$x{!>=)|DrG+FE`Ksx29;b&_#t^yWPJdhAHHq zmM?2zsxou(eyu3?kh0kHMKkvw`Mu-unwY{6B_C z|BtsVzn3eZBelR5{O2cG&->bv|IT)05t{eE12P|af}IV#&zB_;|Ja^)aAmvq%ju;h z;6L~on6xu3*Ozev2C9mlpX89{W;jALfzi2>k;iKd;wVd)^*ioe$1h zd){|R6?MbNTAv@*zzm1aZf|nC{obY@YA?rXe>E1ev2)_58M2ptGsYU;oB!HWxAQ%v z52hD;jC+BMK&I31q{3A^iC5k;WtRtFS|ajMGSXX&x`)H<`0h@@p$!Xpfy^Y)KBpDJ zKi=hvuEy=LXOQpQSpCbF*cu&9gqF5FmWKxA%o*E|;!E?wkEOEx z?|5{rXVMK}##QeN18A>J97vxnz%mHD7@ki;Zy}+gc!$^=4q%Xp^FC>?^zRG*rRxRr zv5oiukLLDmB(Ts|CO^>hX(^F{9nR5~<|&^Y5`~0!SNsi~=2cNRj+ocvzecJIen*4- zUSmdm%aj7d;p3Ju-oS!7xDoR?HlZ^B9M2g@39w8VkZ4=~mjrm-uHsN4*{@)d)@ITn z!qA9s@;}0{eCf!f(TKs>Tv2U~f!ouw6Rf1M-T?o3tyrM2@IRyFf33~_+)Nvx9Fg<3 z2U1OUtjp2H!3^s&-G=Q$wxFh^1D8U!R-Ee9P8QoD=0dHMIJtud8M+d5nc?6CMHz$% z-|Oy9>+8sVk3*5?ZJrX6H%mv>wg9zy*gezNe4}?9!_Pz%TbblO_#(Aa+U#EK$_H2h zW4*g)c({3vI-naSJ0cu;w=@1B2HX)0>E$Ra9>ejXG0hM@!W?rO}^}Y(E zAHtAQNO+1kjk+&}hhctjmP2THSIlDSO3z8kE)W(Xb zW3M5Uw-yU*^rErJ8j6rk@2i`HwwOD!?#}e!ur)^mtpvr+iaFEj!CJ8l9#;Oro9UWP z;g$DJ(f#e(VuH+i^r&$}L}kFHx9H$v@@KSBsV$f~rLQ$95PYOc`+9uWNl2S~;(f1eag|XUk}0jK8un z+*~+q)+awDD<0K^Nj1m){!H2@#QiJVFo+`DcC#wS3;^oS6PW+vfzQY@k zjT(=&d|JBPa{;2fOWItZi^kHRJ{s~9syMa(98%=vT!9n)+3M1ihwR90vn{HaHITmt#REb7PrKF;OV@PkWbxAnVy3$&89`?Hc*|NMuUpHDHzNoG7 z62JZg>Q#n%^#Rl%I^HB2RgfM&7wdu!M~K?Hwck<+B?O$_UPZB;hesPnuWnJ(4kiz( zCUtYe6`7yb-sef6P%!DT2o13B7saOa<*HIc+q6qJw08(AizS;oVfYS@7wo3@^>i;M zbO=lsJX2pP0VJER!hFgVM0$4#qJga?%MU2@4t9LZrMuX|oywFMQ1|N80>%AN0q51q zm!0lI58fk0Q?r;HTOdC}Huo>IoDYKcY@OifH~V{}>$c$rTiTci&fW|+ z;xIiKieQ%E?b<-PApsFtBMDm=tIHcCDA3Coc`<1%Iop*9I2YemYtxXk@62*0z{Bm^ z%rarQM1#IOa}Z&VS^yjhFHF11d1%DZk~^-}OTGtbw9Uv7NNS0inoVdPwL(3rIkexk z>sPaXf9N5y0A9a(kifh3!fynHV&j`L7tbc(t}vc^t9OJEuuhG#xx^P+PVWSf3Ljt^ zL4IU1w=ND-dM7r)z}7CSNXy5$VqLdM1#{kp?1HpmXIW8CN_lOprourHO{>MWpxxP9 zG%~7girESmoLH_7Xz%a-vc3biFn)w(fl;>Urr~hY?OguqHF#_|tC93O!+t>$KgnN2 zW6Xq+<$##1Q2q$XZOAV9MC~D_$`w{>;y3Rb(ty5=+#}AiOEodAbGqA+2&!NkD=*5l z-YXqlIoa%&oZm%lQ10U8bdauyP;%S8{0}74_up_P1ST5rja4oMLKK$z}sBmMkA(%ejEheu`nHATp!jW)1 z2iro51N2O(Bp50*F5@YuG8%j#v_ZTb1>#e>p!Vtlii|aS*-K8T*q4}NG zhyjcnLNKNaSCe>+Z@WY~;w2Uw)dgjf=jjw4k3?_xuFLlh1e8P%!5c41%C@^7Eod8F zeM^7Emzqw|V=hXZ^8Q-7#<%tZM=>D;+s%V9ReBGqnee-DQdg-~&E75DUlajNIz7=oG%P26=9mnK`g%iYT(A z-|DF7`7&$%ZTX?0Mr~a=C0WpNs%!N3(S&VR+k}y$#FzQGejv&yE}Tq`X@dyUB;_e5 zdZIhMb8m9g$OES9h!SL8<2{hMjVZbKgT-hS6qF9Yiy859xLD9;rzDzeBql1WK4f;r zIu0gYp8Y`#Cthy9MtGUahoY&%ttMEP1Z3G)5ENApPsJeQACa0WxW zE<}uuXDn{XoL)+~_c+RZMbH{wn}zZM{je@~CDve-7D905 zD=jap@sr>~9nEDW%}25H&Uy_7@iNn$yDw@r+CGKbGG3A}+kcs;EG}$i0({dY zVYY3pcWtS2PfrRp=vc{Ph1FdS?EB4p?n{FP$DMm>W#oP)Zt1H7!+&OlsgEMHzCXLQ z7~Xw~39Z4xYxDH8GXE&$!Jv%^3JP>LPr#gPHeM(A@lzP?e$Za)plPvDHn}Rba9xHVsv@$ zU1+%u90-WN?FwYE9Bl_a;nx)0JK}d<$#FWg#+-F;$@SYtr0+2It(p*|bq!hX2EGlv z?aJR6acA`X&T-OjzZid@Esm-dF9P&nc7%DK6r!_TM;_|%ZK1Ov1eB3mxtILxb8K#? zfJRBzA^B$XY>Ezm_D+tQp)%C^`=$ZY2g;B=iQC6a7g@1cD6~R;f5W`4*UUu@WF@@E z;Kf9bcM487IFUA~2d~91{7?#`SAwI3NkI|qKj(Q5$fB+zQ_uLd7AZ}6fZ<4VR|z=N-#&FiTv@0=ckA*>|h%Sw(3%yag*@OfzOJ=r&~`{qEI z{l){_(l)_Qxtqz(br;cGqi{@TDZIFHewG|0wNPC77Q#A>>@F@{Psvs?_wIvnSeN1r zsRNwl!9KT|H>VyKsESF=_ff!0STxjOAN&XQa;0U;pC}uI7!Rr(Qh|v8(b_Isj}V(# zs)daRZI3n^cCtt!fN9YmJnDpYkNRmJYQ^i$uXEqmnzr0kGT(~)IS^+m(_9H^PtBn% zG+Cn`nkv)$sk4Tk@ZIcdjiiiy2OiRYUIPf6glJ>1PXg`IZj;p^s7(KgV1Hq{Tax?f za|@q{f$!sb_=JOVFsZYhnDae2+P_rV136ze91`t0sZ^V-ASX}g1udJ#2Nu^2pzp

wz6v7kMotpYxDL_oMQGbXPpIT2{nDKR!j5+>ixU35+ z9m1?&uC^i@V^3`RyV^_j*(iPvx89#kxGzTh>T0b|N7GcT1@pgiexVq&n(~Pruoc+9 zFpRR9MI69rAsl9!@QDgC-cP#zpxUXwju_~@OHeE&I77UY2$khr#7<7Up1Coq?scTn z$n;uyu5l|N25X&d&Zgkjn? z%uCTbt!$^|#AoM|0X)g}W*IC6-G(>h>D!e1@`^FEvzN!2tF1z(?_>E_1*7igL634zMdYA;%KA4 zr7(Ma=h8-iqk3o8zn`dam2zpn#>w%0C{%-*l4~~8o&j#OE?sT4QaVJLk4{@|g4E?rb^-jwG3JEcSue(#QwJFQHv7v__#D9bR^;L1K9Ccf;V zx7T^i1Fz5UoOO<<3ExGs^SW@<>yLL^M2a#V%l%qB@@dyuB;1j7qv;CIB-QUty1<=j z=EM0oSgN&nA-h0C(nRv;LM28~Dq=bRr|&dY&o-MsH+)qQNiTg049^VE5Kt^hSS!Lb z%*4(xGKktCSKce5J zYSB63JR{}DSE1HF`V9HE#y?`)SJf_+t|PQMl|;;-s8k)Fb!jwgYG%*dbam^VIE=~H z-on`Uu6%1Q_FU1>ieKJf_yv$fL;HBhSf+6fq zVZS+?t6`9w)s=TQD5`0zf39vmm!AR$`q^B$7XKRGQzlX!D#bEiDk+FS1D90%%GRgl zja3YDs6&u z1M#|%(niZBHNwL%Vbh@m|40>S+13SuIxKt=1A85(;cd09Y&eVwM@scw_dX_h^xvBm z-h6A}L=Nxx(k!ry+5p&x=W}M&Q|WDh44nOLryqWlsl&x5m>2P{oO0ZML^zG7J}0oU zFQ=Kl{1E}cI^SB}x2K8}+VA8fRdUPxV;Y}IWgD{Op$L-hjPU;@`^OqNUE33$b88oW zw+|0bh_Bd&#s?(uOD)9WQefc-r-xn3HwtgiO@!~O4NKy7)>IUu_-<9%{X~2#IH#$- zSZMaHxS@U171zkgM#$KANGvUwJH<`)k*@4(HV2Jrj+|>;H41mjqVrO*N67vLP=kY| zh5ge|G)lb3U(OELlkv?gPG`a4!MOD8*v%`@Og?K zxN~15*v)T|@Oy9LJgv8L6zd^P0=(vcTh*RLU$sht14(7ESrXYfRj*n3sCCxQ`cY;K zOA!^k)Md12TN3&CS~&DM5>-Bh5p&~OM)d9v52+p@J&(C9YumJYq}j^%vI_q&<20g55_Y?0c`if+a}#a(rcN z5OlOb)#Igd?XU%9x*+0?$b@w-XVgc`3Q39ofQ5l<4}=oIn(#$L8TENYBcD_8{>Osv z2`|W18AMjFRon5HBJ-jg!rndhZGc|5S=TTYgaCa1gb49#TI4cL5UfS~uzEm;xuF`X z#hO6BpBn*%Eo1q_s$EbhTGFM4-2ZE7D9M!gUWnCpvqc z6NY2h(GF~U)r3*a{@d)#Fofwo>o7*eDEzY|(A52ZWttXAr6J?{k41QKMgY`&NSf)G z+n~?hd&dG0H3}JgQJ>UdQ(9OL@=S|xGN!uF!!>j;797By&3p+-Y-MMS?Q|saM3%z$ z)YcY#7QXk~=J0daZ_gbo(P(h5V-5HhlXAp!Bs+rUI9$?bB827J)6By> z&UOnK=^sew&1w2CICKg4(VvPX7=r7)u%=F(_w1Sl$pXGMy??2%M+Ou+1Z2TIc>~$A zZU-QN0E5q|(PK4Z&X$}cOv{9HkN>vAE+ZBEvZz^4i(9KDS8C<9H~1GQtx_sbx-QflYIE^x zc|NrLDb6`GW`B(qwLBrgR|D+B&C4}Gy>>jLkfg|OMspjG;+Dk7i_Ya8Cb-#@GxJ+V zSocO;qW&Po>uagH?{icF^~PVk!6w+c?KAzX%(+V*o!!^@xAqI3oz~!?@-#Q!nR^yp zuiW%aY^i^^h<$~A(ILHMTi_Q)#ug>Tz0Z*zHS+r!?R&zlH1qTK5aT|j^hQ`<(}ZyS z_6tX6{kA`Krg0zoiqCQYa1PVqa(x3PqmH(s=9XlPMj*KKUH4yD6blGK%jtmQOX>?v zJWOlaPUeBM7`Nz@yOFX8H~t9t#1BqyAJJ&n7b0s#e~jz^8#O7R$8 z;;zipDfNgg+{ocHR{>8td-?G569GE^|tWCiooAl?!Y7LKK;`&X9Xd1fTh@GG@xyOJwroVrM zCh#e}cKCkrs6UP{E}Qj6M<2a}N1$~5zGle)Rr1;~tw7O5JB?!12=uO=f}VL@h~5xa znsptTO=6fG$_3FS5IQBGsR4`TQdU98{N~DMs1SF zxLH!cjm6^4`owc>M3rJ#;t)}4=-%EF2(NI}sV96z2vzJhNB$3}6TX^!bUIumj(oRQ zB8VLjN{7R97=#(=fx<-t2ReHa+U#ABfOgkF7KnH*aPV#pv-a}f>$5*`)_eSZV+7pko#9PKt7L;ni_mdtB zp91d)8Sb-8#d!Y*%M@zb1y8zLXxgAIU$OhBJ(_CEl2F^CsM*mhg3byfj>hUSKZm#v zyf4Kh3i6|$ed-FQf{%NIi4}t$MuYi{k7b%YlRau@B@Y4Kd3v7t6p!pqiU(sX&t!x- zSmxJlOh`8>S67VL{OO00niK=SgK(|;e7+(#_x4=%quuz-g&U|J3!ew^z1D>|EQHgr zZQU6A(`K!Olk~q-Vap8^TI7#xDmN;J_;Vpk8vSMZk|8%KhKbLGT$~%75lTsJjVoJ) z?PPkzl*~RqA$C$~W1f$ao)L;gm*7#Rgd4T3(x^NnJHeC;$=RqJ=3xPDnQc%GF#}(( z#MP+0i-Psgo(Wb>Ijv;3c34Gqe;6WO72TsvU&|Kh*})S=J5NeP`}4NM8; zIL`QgxQpfJ&_fk~hiPqrE35DChI=Xv_qVRA{@*V6?(*QP_Rj{P$G7a$$Bc;WNK4k6 z5)V?zJMg@2tKi>s=A&}P_Fpsoz4MEmyW>VqVR&v;cm^9nb+1DZ7J}&5=#X=wq#Cym zIMQ+7(Iq*Vo++~839r@pb#cd*%Q=xH-G|tb%|{$9^66tzLpf-{3PCC~&yO($EB8x$ zYG{E+6I1fwTTefj3YPa~`;_4gE+q$O=cAmTXCyXTFXt-$l* z&J_m<6x%;Dg!gc)ze{rZ;ffO#o^1kHVhm2aQodSLPkkTb($&`im=>-($(;h0*a)Yz zO@`1ljgzDMJL?zgNEjghbyEnmY)dn*d=E7qRZMTBsk|gH8NGG&FdhAr;clf72)Q#> zbE}`~k&47c776NcPG^cEPcyRH`(EiJ@7+eYMu=Z9^oUU7Woh?fe0?eXg5FOrMU}k6 z*K~E@f+#*g$DU?;qc^DEK&MIYQB&1!H2OB2O;7q+!bMx|hbH6~y1FfD#}>u;_}qLv z>IV?7U!;Tv29~qW?(uYm3Dnr5+Gc6g4cAI44nBL9(eh2kUo~r2@x;7XstHI>WTFTd zZd5$yzhO;~;2MX&kVs7IuN_yJuR||$otdyvqQ$>LnVt_xNmgb|)AU&EN#n7Ow!fe} zS^l}?mh7GR$Rkl*Xo*(pPVK*bdmdfF!f*bmv&(+l3^uIvP|YGf@5AzaXJZ*bbm=Cq z)U1+XkzwiN!P}3AaRDZzGe4hSl#N?##6`2;2p!q@OlMpoIXx`Qn)EFfFAv_d2)C<` zG#}>R$NrJKIkO48P<4{*&3xMRv|*87*nI+lRWRA844b1?Is7>(G$zm^#!hq8HLB@c zh!aR*(MBNs6$*)Jt{t4)f$b-x8^!G&cZIGKMSDFiii2683NaiN6*xyu4UZd~n;3(z zfLsv%NxSOgqU^m4x$5Q>yvakXF~TXt>2)tQO`lC${<^{_zK zJ9IO#+HCXmKd2`ibk+aaS%jVK|IS;4hmVT`Nm!WN<)^c`u^p1f?`ngKG4l?)PZ6z~ z&2O6TACScfE?%@fdT94hve*buLU&T&7a#T-T^AnjI>ky8ZnJMe=d`7BBJ%0CM|( ze5R2-E*8-~I{|<_|2~FA2=+ejr@^4zXC>DQ(xL-^9gC2tCRwyAlom#8?>(ifl88*s zN^qvA;VFe^5?nrXZp}!`d5_2W`Dr9&GEPMxy$Ay}0uO5n3C+j>Ij$8l`{OpB#~j~d zZeO2t{qzpB7&0L<|0zC2N@LHwxglb!&>O{_ZBD)}t|%R9MKp&=^Hj1Zx;Ml{sjwgZ zTPSKv7me^jAqXYe`&v3wn`lObrnO`f(s_??w&WxHw=XuC9vWd{Nh3;f2iO0xKcr*& zr3}MPoWEdy^Dn>EdYe>H6FfyI1Jf9@XcFA{s=#i6f$3n>4bc8EI0 zm*u9WrZXYLGKVF>Dfu@Jg71i83+>nS%R`#xQ1H5DLMkOJc`||JmEqm=v2*~)*9zmb z(&NvKus^?);927uC*5Ki;?v#+*aXP52z*u+oGABhB^ zu4!n-peI9z>d|Hu)^5S=uT7@Cc9mW%iT~ta(a?B;@h!GSFR7d9g?KF*|9zUGpuEJ( z`)6E*a8)a?TMQ|jOZW%lgcieEQC4npi9TDT;`~>9cfO;i7fP={8dXAQNM zZZR4Fgu6nVPS*VP91a2*Kd=v8+l02_7UsC}Pr)h91uUCR_7lcl%EVIQ-||J_3BeDfM!1S+OU#32 zi!G@3SnowJJ5u;bVNh(2Cfpa)1Kmu|rB)5VA08uU2BC0%bf|;D-TfgJ9S@RD=HTL_ zuJZ!FTy3d#$FjpnI0LfzkL5{vP#Q(8(kBRBPv44|IG^d@O{>vd0@+cz?E$O_nbm7e zK6oHE?02RJ-w@?*=i)AK^#vzP2$ZIE8k;%@tkosO^Qy zx!DAcwc_gQeVru^gzzf4cl7hT&g7!N;>OZ7D+)T*C$$g$qH7gv7_+a5FZ+hCxvz@g z9`v>lUeKx*jAzBhl3_6*@{J^_Qf>Q_Qk5{Fy`*3)sSlvaOYklR?d7@K6G3*rt>%MP z@k1SBg%ocbE<^4{5}OV`AT_`numIU_SR(E*%M6c9>5jmfX$^rrZrEH0fqUqNwTNBy zklr=V9S|@iZk2v$B*>CpMZgGUv5E&>hLCO-13xLriq{|@zs{p9^U4MT*~>%C{YB{8 zv8o>L&CE5nzXXNnkqhrlYr91XddTo>du&LKKE>F@SDQIy0Q6cc=j559lNF_DLHKC+ z^ux1aV&XIdyv)VN>#?F0)tdZGdX23nSY8h{6sR$OLiYB&QHGAiQte9Wir&-8y`H2 z^>6p+a%IJZw@AQayw7%(QsvjITlJ707u4AQ_4IZ`%2@{W=Ab|~KAFb=g zkNKYGz4^f32X`n>AJbRZLK-FC^&fzHmwp*vGl8$k6kaDMl3Aj($taqEQz#X(bYH(= zwG&>@Pm#vz7Df+}XP{s-c-U#dqxOV!2JLJUPaMxGoCGIgoH8%=;%mYVKA=83XPf|SAmt1D`E6GArrQeN;vV=QLL++kO z&-PZORq@wHSQ6L3*)bMGYk5fq(}&JngLnm>sYnBJ{3zt>E7(C$U5xq0;Z`1oc(My6 zUuj{iG+j;Dv%U^CM4mGi_uYw~aot6FC=xvlPE85k{BHR*sQ5y@dSaQft4_VgqGT0j znuz{TC4>Uv+5TDn>`x~7w~lwJrJfo{hbsP8>GmOC>kyOj=KU1NvU_N7*I&`pU$2P(#6|@L|Fa za&F>bVi%MjB^=_w8qdSgaIxW_4DeHXVu*DW-B4=Es*UL7ADeZXrZe&IENKszO!t} za2=USI(m5#`_?tqS}f2Ub%uAan2j3AVFh2W8JfQ=z))FKb~cCaV7mXpR~moeLUQ8v zX@g?&qC>fZp+H6zYViIxQe=Fji6N^aE8d5~G`n1(@kO{!0=3v!N0f(A7niUCFJ!kp zbQpQEl4*VOf=5&(w>sXWfFZbWK{+7un#m}O#p@pabF>xOJQVI-jnVb&OrvDI`1$af zoQ)84$EOdVxgI!~Qe|IplPXWQsBE+`#A}7n_Ycwi6+TdBnG?s_u}%-Y|9Y!xSW;`2a-4Zf2|t z4I`}*OabR-Z&a0k1Zp!_AfQSMrqAELsn^NYkG$^Ijl@>~be;mFi%VK*Ntjr)SmMUlnEOo3Z;+Lvs)wta`-NN1#+B(ZK&|;gX*&BM_Y*h>&OhaPlo?G zN{=d#(XZN&)SA4b!R~M=y|wNh_f%|hII5GuzfwAtMO>@198fWsH259#wRu8gtyhT< zZR{LE7Kv;**eGjtKfe<9t{$W0!pCoxtEtUg;r431S2oS&ZT3+7}0sXDO_!ZK#!22sG>kH?eMK4CMlx3ID_FW`&vskRgD~9y_hMKry89>SgjfG(LR@c#k#zJccj) z!$&fQE|yiGBx49@hCBa!+c^#BW}24Y8^4_ZbXUEBfkQ!XbA!j+b3>%i51-;vFuDUK zXKbz2XVHzhT=oy26)y|I+#f7yREzFHJOAKyEqgztsZWd=cmMA2{mXBa?qgwB<6>BT zM4k0SOMguVpeRCE)xZs_??5lA3te^~lSQ?VbKI7#j)iBdmQ*&;+wOoaX$Ho385)L4 zi;%?*%T7ZpMF#C)!abw+E+)89wjf_$5?r;)#?>067Q%STw@|YI=j6*?2!uC)pZUi1 z>V$7QMaJwH0ftIeWsUvN_N#dD#dEyb2K)JHIQ`(G6+Ib3C-~cBftk-r^sBm*WXLj2u(1a=`K0NHn!!6*cn5|^mm}Ey<&;HI@&H-YXwFQ-IlJ{HkFoC=y zF*&0zjXO@#&x}59YqZnfUdw+VU}0QQ`W!qe-(b-Fs0MwRT^!G(3bYQfdmSJ?H;$qM z^^T+G*tRqKriV8U8P1*ayy-O?QLAj|L&AU$a2b^=LkR{CXgE-~rLUF}H3a>~vFIJA z7vx4OOL!fp*?FvDpjxjtxzgc zX}Ntaf>)FsvxJZtd@-c7%Mp(q; z=^`}}+HnA<(0&ck!9J{@zY4~lMynAW{p9P;q{3B8H`59f_Tp;0uT80DdhbJsgg)G? zP+yDs#3cu1pbjFCfdUYN-j>sJ^-KOYf>e5=wGrY>1dZ)6J>5)&*+L&(r^CrTDj&{F z)XYpXtTq}qWh*ZT>JOJnBF?|N_1F(jMHz}e82vE?4VUsRY3m}o?Pg~4QYsQW1y zJRBuDITW_*WO?BcWSO7O9s+x29q!WL%6H^7TOqiI<~Cs1R1tywjsW^1jG?P#^UH#d zWoHkv#ya>MD@16j%xy=Nz<9aHSjp%>5b5iFhU0eqYJMR{&~}|zhf7tNA^TqX_#vxR zIR~YgX%%f85B?3GD}r7~O5_$rFEl4g+|2cy^YK?+*HACESXnD=vydy%A|xs783Um^HTC26%?Dp*_xC37Z9i2P6Cs!Z;2+D*_tN$#hGf;uvy> zeBGo!GQolA!&k@&B#V0Zq*sL{-rw-Mh>VM17x4`-xl3QmKJODKRZwxx3YqeE zc2bNh8u9uuBwOoL68SbL@+mockAvc2t!@KeAUxK2bTxU=IHVV9w}bV|F?#x~Zauob zW5PdD%RMW%MFtX{MW?EW=D|fYS=}1Bwkvhm2y3KSHon!VdGaCDDJ|u93c_uK8}n(l z41|-6jVr8wU~`c%4F_eLIUg-U3L06uE0tc8zugvM2#&TN-+MtN~R zW4E<>>$iw97A^Of2S(`3iyz^0g=}<`&ZaC`Y*0A>lnr9;>&SBadci7*zolvwzyo027 zk0JF$K2yPPt#UlV)KZQ;jvS6DnwW8qas0Vtn{q1n=6gx1Kbg{K);r?RSTVcaMw~A* z)f_!Fd8Dc0R|^p3hC}2Ot+6a+eNAx>|RXVJ~6n9QV$F zECkf(LJTH%BbF(xaq|~nR9(V{Loa9_ohNI2GTcaRYTJfGY%%fvS6zFjFU7{wWa2f4n}%)ZpqUh?j*;9S!i zCsgLioO${jCqEOA1J;`yf|vxE+#h8K%wRS`(EVGBdo2<*pyg=eGXXjeW)}w+-SlM> z5SPYO2pu;w-Nc7K8;}yPy2Ue3H}O#IuPJHrjUnL;i|)U3{l#rk5eBSS^2RZ?sh(x|v&cxDJJ$W&syFpk~uv09Hl#8#VmHkyz! zkSd|}@O85kufRcPUmJx!W7kV4nKAj$?)X+WGxG)8VsN$i>ATBBl9 zyG)XtIl%L7eZ8N1Uux|!my{nO!&}bAH-(=DNA$xoEF0weukuCZ?g0vPDOPKb z^WyLdkhXls5}7az2=3VNjP=}kg^lf84`_8KMOb!geqP6U3<1pW*)mDwgF_^Bb2>D0 zi^pzU-?80hvJ@FF?Dk8;(H!%eQ-c-tlIUS~F7H%m7Kg5ME)}|h-S}?v{(mH3chF`B zztb#QK&iRdoRS#Lf2Ub(`M94ERfEt&<4>d&jmj3ljm}Xou#=JgyPl;BUk#Sx$C^J_ ziVL)5=b0Z4zlxqOyL6UAR@o1wYp7bh8)3-}wicE(VBV4BJEGWrSnU$W8@xRPgoYUm z4=LrhRkOO?AA{)6aseo+&m4vuKr@T0V{!$ZapHU&_igbOWm6;&7-(Z1hT12Qb}3A z`)?oeX3W_zGV0e2mOWp|Vyb|>4_`@7X44IbZ1_`uC?+;*-gkCBf4mz7!pt(I6PDd9 z!2K~o|94ZpUpn`$+F!nrz$d{w6k+}Yz&B!WKK`&C^lg)A z_m<8ao=~vZUB1sUC(FrL2s*RU(CLM(dq#$!mC1#U!U{^gy^+~c$;5qcW#&!dw@{_( zk2{J6-sf(%GJj_~WCcRls9lrc!YY+K`ZyyfWN3IzKxBPOz%xqf*pu73NlGjmEZehR zC-q(5@P5<^kGq#O965z5VFmwl>FPX6V=eHZi@G>T&*y`ilEuuY?4@v~_Rl-ks{$dx zD>LBSqKxR$bFL1X?)}-WGByWrUPMnG=S+m?%>hfUR~c*g$3&^5?FN-Pboy?xDHOc} z-%BK;;X1!UWJ}!C_hd^vYo9eASZ#T$HRGT?e>9V83mNgrUzWg}N72#pY>wRfBxK^9 zppyd*D|oiev4!f+mJ2Jp{c-fG+=_O4t=+6S6LEV(Qv8P%Fh4H*J#}G(^s3A>?bq;3 zsbJ24&Z|lLx(Y>4LLKW$*m$cdyJWSAH2Wvz6mrke_*%&-+kM^N_#BF}){L8-EA*h~A5TiS!c|O* zDEMVDoD(8H&AEby%sxFkMPY%?#i%HU&w4SHm}M zb+

vQ0ml|AZ@^j^#j9@p%p|e+Qzip_^XQ`nY8SlyLBUD$bjtfqm~m(xw4P@%q&S zK|r&h4W*W~fNWs^vS*I_(10h-apHcqvi@#s0{}grhBcx@vUW zZLV&~@toeEde>X$S#moEPQ&VYqM=x{@_8{Sbfxa1{g-48C^#S)U8JdVi?VhXSw12F zJLO0cZS~L&e>0%K?o=c~;vnsA%pg{Ou(Yl&C{gib{OGR2U+V$QF7q%~lDnXqvVnad zXT=5S((ju={>>BgX+!lc9fOx=Z-01xE^iJv31^ z55HT&3y}VGhodLzf6(<7P;mx9f^cwm_rcxW-Q8UVC%C&i1b25QVUPg9-GheUPH-9A z$xn7~ckkZ)_ujnfuI{dy>aLpaebd!l^(^t-nQ#`Id){-f$1|5JvddQ&e9lKkIP9t- z(qb~`yaDFbx^(7L%3WYS1X^$u>0LhMa^22*PVD6&BB2Ok#C(+TLiPI_>($=1%4qDh zzyKfp*Eg{f2}#M%X(OF6@j_fB^Coh`8t+!M%Z#-^Vk(wEb&+kXRq@?{r~@mZ)a~QV zfhe!YqCVS<*SN7yew;fWHw-?G>;zk^9orvi9{vN`wGbMJbIZz|)SYPIQmQHqT+|1*WCF#JT>; zr{~Hx22eD^g`WB3X#q9au9@WjXtEJe@x@?xvgn*5DOSI@JGz<9 z?P8sEx|alTuhLH71X_EjrZYY(b%9RkP?S4>+fueZD4xV~O+rHK>vPxTCH?+x-YraM zXa@z?{X{J6-nK=4YCe|uBE2u9JOMB-Z8J$mPl zxx93y?>^mw;~Lr@tV#iLee05lLQ051)+MAjZJ7n}`kGA}2n1J@09hh7@Sz>K&Twge zKSp%TyJQm>3ItOF|44l=n)@~=nPOV|R_~1yP0?%gjI_asGD$yNm^Z&q|L7a~xeaLT zydDvB<#8TA!2bHqWP)$}q;?c-sy-NBIHAQy^nzM7_9f{L_P`LdQO zX4I2-P+X+&SQ1)N-0P-RH)U_-6K&|$=NnA0ro$dYP~G)MCG#5U^|9JzyGm&U>b0fE zX}f9;dAo|K0q)Vx;RCt)5n5FFjPjaJ{rIUjLk2QP!4wUw`Ro_6r9cTGI&3Aa@ zQX>Y71WF>KLW|I!k**1g1T&f@C@VB{L^K$r(+*WC>bbV$Dwy`S;CL)9L}MHd9>>OL zlB5vKSPWtbz>u#rl#6C zSCsC0ZH?DJ($;*-cwW+jbSn0H#RIkA#gR!6af}r z5Wj~}krKkle;oUg*E$Xih53LK?_pGyB3SZDzLr$#!-5x&86!c&ln)v9!M%!UKO%w# z%_@8&c}^;KiEPUV#^Cx#h$RRdw2tM&U>s(5d?9!?HM+x5ibR9Ha4f-WY+k1R(q>1* zx=8(%5w(j^bfLLmXA$kT;Dw;jsQp?Pbvatl%sNpO7|~QYc*LS+o<8<1VEN@E1Qbuf zHCsD-97ju!**S=NY+L5QKDnC}`y-VVlsKO={N(X}TOf=Ifr1ThJ{)VHw^zzSv z8`L!ue4^Af2BxGl(XZoeOfHnSXkCmJ_!!^3mMK^cI;lLZ2B+mt0)JD)Bg=7l$Wf@g z(8(Lq+a|cizU7^+e$ru@T*m_l(zC_L!~b+nuVfQ>4J2PuG|ZTjH^V%h$yUz$)YH1poqALzeEHbhnlpu5$-9ssMjn;p=BS;E=DQdtY}BW;3(v!$Jr4S-eN(#pxh&cl~k z4v|&L%H6`s$r9+~@sGwGV93t?@iiL8h^$)9+D>*KBvt@+j(<%bzWRSM{fh(fA7lPk zK#2cclz%M#F?6$Y@o;tnu>X@-bf{6i`$dPG)LH%lwGe^aFUuPJ^sYI?f3I9NG; zOc}uTZw{=w07Eu_Aul(ZF@T*HV93GC%fiRWVGQ8p`Do?fU}5LsGX`+60}OfC**<#s z`2f8CHH7 z4qfW%6KlE;dDuiUAWCsBkkx0qw77|(uDZT`7Y2A>xN)O|D&_qUxV0tPi^lm|b8$m^ zV^Mo?af4G58uAka2pM|J*1?TOgt$?N*sPkUPtEYM+P&xani$454kD(&=&-$2oebTL zFeF=D;hST%M_7;Z>}Nwy`QpoYl#kj3sb!=z_XRZigmQO0pnk12EBV=lB4(PWOaV? z6CPgV`!I?<0z*KCA;&6PPhO6n^aTa^9%PlvVV6%!X51|Zf}T?op=_pa^Et6U9{?I~ zLtS74YDfKO`hB4C`S)_n>9kJ#P3O&Y!e@fTS+v5Sv5}KepFY*bMj5o=Xwy@8q(d5( zDEeR#4qL-Z(`k%Kx?cd^)@o(vn;zdWJ}O!%Wd*-!{Yun_|( zBq#>7;dL3*zGPX@h+?sf-fNX#CR_V;pL{PK5fFPO{w=%x*v46aLA(3- z)4chCYJPnA0ouVd)(3X?IIhQ9C3xEM(P)KOmy__k8T!;&GCK|gyTj0emNp~CkCeGY zK@#F;hjEG#CKbZBKVFd)LgY%AC~A_!3-;X;@;i)pb}wK-o~nF$gYY!J5eNnI4D<(e z6rkC|iE(1X5B~VOoDYRDGTR0uUL77hpeA{EpZY|3!!~Wt8b+nl&xYdDR$AyguhLF5 znFIfWI5#wgs6?DEL1YpA^s(2mZ$8Z0XI*ytV(@>S#%0{~X8&d)@)rAIZ~BR1$j}y;sGVSGE;Q!zr9Z zdC5*(l?{<-w-e(2@PuQa9t3eO{m3UG`p5>^IwP4U5(y?AxKV-=1@mc!JbcH2?pS<* z+GUHNi)h1evbc&;3!RX!5=!HG$_=)Lae957Zsq%S8u)-;Oz+-#Q+~JAc z(0lpPJSx)V+d}sMx!$OG=5QVvXD3L{;xy$e0!-Fzo_%eTjD~s;dXncCY0>+&QarK{ z|Hhe|+BG(n-cb2nSvGkeBpUg6jIG3*wFFN-vMclh5O#L`aTNK{~^|yPzS~iL)GR*x9JwRwxLWTt7|X;`bD-@d<&M+RkRwEMOJbVi`@`NOV2q z0xBzT&=ZyPq@U?L1^v|>=LO-ud8P>v??z%}6S*=yLtLQ2*9Q+_y$sKVWDAOBzAlqB z99rOVGF?(+qRZ}ftc8Q{b|_)q6Tt){7Q&{z?YyAZsRy{JIe7cPs}tcBqN0XDnW47F zy_+sRWf4}9#lz7*Pub_pOuaYVFiZ3&jlV^MGs*N5E%;Oa@V7oZJ5EsPtqt>+WBoE@J2HUB8_VM6(0>UE#(4Ezi|}SDc;Gc>`o|Z4!2%)@#VD( zVjj4^1-)eN;nF~ZaI=tvUNK@cFuDq%`}u7Bf?}(ocl7QGWn>vykD8#n_xdq-;1p+i zoA4QS2@fjFaR;3~5b9yaz+m`%Z_s}E8KWoE1(%Yq_xM%lgXpnrwtN+qx2}Wz0(Hm4 z1^(?GW!g&Qi_|lZYu*FXHLf{X-^j=_{cFI$J$>f`z>xLCiLLIOPvn}7e*k4-5(k3bu_glxgri|G-n$Ancn{cn)QPe8=)auq z?zP`vga_=f&&DqL1U(XFE_mqCD-E;@#`;Hvxar|)0zodAagnBv@68(X;;4u5eSxox zbQqIrJw&xbsMVNVa(NiPErd$*f59}DD+t*bhriK@$rgXXRnW3eR(TNVe|*I6^`XR% zi6>8+yo35yN0vQ}cDLXwym}A&1+J2H{_W1H^g*PDK$lULIn2H~B+AJu@A!3@yLY4FpA~zB;~zCU}-SzXRT(VACs<^d>?o zcXQZqf+qidvW#6aC!H4ar+`N)i3nwYC;UZdTU-Pbo|He#xax7wTK*LNULf)pbqr!` zo98yP`!Dg^yBP5mS>Ifco;CiGF5z0y-!_5wwvcz@IJT^{UXrM5{Aa}T23Y15xTRzZ z{9vLTL|BC%VQV)KMh|gCpdg$E(V1X;M#_j9Gtpks<3Gr+e{SBMFDI5S@87qRphT-V zub9FBL1C}(hgq(oAKtFGVLb*`)0sZg3zCNiGXcwzf{&a8NsTuX3{Wi`#R+;nM0)v4 z>_^P1qwb3yrU4vcYaRH-D4`I)e_DmKLcxx8hKY-IW^#1yW-H;ozF&<#J-q|{Vp++z z@O-QbQPmIkHXn7aheNW9cH$R(mL@Lmp8nGIeZPSnsDp2K4Hm|o z$ z+C2TvjLUCR)`zIX_)C<1hPxG2CQ@M2ArIr*O2DXCXE{y!f}BFRwc! zqe*i`y$mjdJA$s?ABYR-7hj1JZ1jT;tPf>jZf@?$g9@o8Y~qNUn>hu0oBjcc&6r1- z<4-3G$hshZkp(14Q#C*oEBvg=Fs4R_2Ksl(Wt)~W3pM=Mmt_q%#vcVja_jqaSMwat z41onfOz7h1EfQTIDO)TjTvF_JhLhp(-NjvpE7U8NtDq}jqvkqe5M5!imm;fzsbZD3 zpLWYcDdE@L+T4a(O=dM_^^Nil!@HOh*F~>I!&=E&Q@9PjBVB$y)GWAhSGQ6jt?A_G z$+AC2zV$``-Z8~Lviqa^wDqanQ-r{TW*UQNgDS&@#&2y-v1P8c?c43M?cnxmKE?Iq z_Vo72U1Ha1A42Dtw$(lvN58y&>y{ix_Eu=aTIR`2`?~MU&0j%Zn>HLe2k(o|5e%ap zlN^iAsn_yap<2-#|H@4%x6mdk*2~o^)vM(H&A82Ys++4issY*@>pz{vH2<&}E&M5J z=;C#~($u_EWbE$srtdtH@9ul}{^$7W-lzKv%f{rHu5 z#yJdF2$-|zlQfuk(U>3D44h6sO)58=SQ#(b|3GgsvRnrYyWzNMue)7e>U}F;ayxZ6 zZ~fYx=?}&1jnB;6vE(*AmH;Il1xT#xryFQ~4_@|0o|=$|sw4 zR9p$>Uwt@it%rMsWx{a&n`ivLZ=7ZzMpoQaXEuaYZ-rtm1^qJb>yPv|_f7~|ZImi9+7{E0lZ_DOpKEewHsO1C)D-`(*NbnBGHvmTFqwzjCU3 zi3*3)Hk(g=p1dp)Ek4v-YHyYG7`}Bp%io2cD7Oa}s3m0lRsd))i`7w*j^{DO514;&5UPZ=Hwz0o!j0gO}Yyn6SHA6cs1cw6Ek<7v6^7@YS%lfqD?Uh)aF2k8tbIZGD98WTeru;F#Gc>o=rzh^lS>sd?{y z1|NvT0dMdphe)z*Qx`(gFxqWyADF@jel>*CV#Li)v|We|gQh#mK1iEUtR;k-p-?X9 zg<(Iys_iJ7(cU}4Uly^zDD_dk+hI>?(Q}4q=d+)XiZODF2@FbLEc`G6V3y21SffPy z@9+<2`ocljnSC7X5Is@OK;9Lx3yO4jDJsnTNK!Z6AJOZI7vb*P50$6YVQs;~q@)ruw9c zIZ&k#1K`F5_l>VfeaY^VsW`Hc$;ZJk#I(`dP>ec{)O63tJ`0^SAV>Z;7LRQ7?UabaOF^Ez!F%i zMxlgZ4bK_L7^wlwWg`iqGM#t2=SNzHqIIPv&JD5j)fLxd4bf? zsWS}97GaK{!oyI=OMoL__%wbF1_2tcWa-wCVcKQPG*~ow6?tW%+(LFHcORYmnEaG7 znWXVXcwKo0Ki}LbX0mhwpG(+j+cim?iE^Tg?8zC1cwf4co<)8Tcw(&_xSdB#)p z??60j_v=JmL-+ego^$ux5)|A{f%lgG-&wd|!PgyG*~FmN@hZHT#Q)dl@5;)YQ`=!e zHZBZ5Zf0((kfn=8DKojqG=m}R@Q9SGbgnYS&Io80UD8FZSVXL9 zv06KoP7Sgq@=6+~Nw9H%nP$Y)OP`Q&R1h>U*HAjZY@t0TM%rf7zp@{i@3mhqlC*HK z*Ye^y!>+~+*mV}v2{5^!Xs%4B=xzn{D0vm_-%6=8?0Ep@br=!HA`fyQSCkp{ zCA{Yba$5)DAkUm+L!1I$eX3ZPalDQolaa+H9OInV#g!}&Xsc^nZc!0)#k3|!a7@*X z62GvT5w^n^mXu8xb5yi|0IQ#w6%{?xt}?FN*U?3%N%9Bu&4kECz7k&Q+yE`{@=)qi zyvx1XHrA~|_;Ki;Yaba#7b~3l)kj9)<@;Tc)+0CkQ^hev!>!4 z9T}~>AJXUFzv|pGF%F8TWv<3Ctj4(<<;}$mFIW;MZ0hNp(D3dibL1i9WjF^-KT7Cm zva5R8?~M*Gwfw9TTaxfRueEn%0|OsxHn0s%LD!YF@&CEB5! zZ5?r}!y^MF-Jfp~N$u;aS2&JQbM=Meozz667;8a8zf3TKuh&AkQ7dGuIA7tvSh~&= zeO|u!hyzW3v2Te>adLH}`OB4w)uyU~<*bx)mk5P}f{QG@*pbAlY+>$IhtVCJ%OW_{ zALhT3u3@Sl#Xm=4rm9NszJU09{@fX%sNW#-$UIvcbFbc z^P<^>Bu6aeH?)^ccFzBu*gu~0BmnF(J7DUU09ca$hCfMEito||l~sBHwX1kSj@xPn z(F=c&p`m?dqgdkD=g<#(KoXFXHRH_LtU;MF%4!%@ZX^k%&=vM-j1@hX2!!*B*uz0& zM-O$piPVFaHKIa3fp7D;#JSjZXlpic>1rg&-9h$*g|+fJw2y3DJ{i#r=7l73G`86t zQyNpnhLwbclrCeD{!IUZ(LEcbMbuG)V!ROb%}$beRV?!GY;n1X(04xav{?OPG0=I) zna#6yNNoEz^5yXJe;CGKPuU_2nzWGguU4x#qAyIXV-Sw4OcgvAPj$UXa4N|ZR|hCo zjpwg`bma~-<KS5IH-qgcEed_Qwm%+Oc?Izvc7d$qPcxOD^?b4aDG3+ooI{ zZYd_4-$;&rWXr5WzmyRozA8jLwa;W(U5thI>}{ZA3scVQSx!m)hK`#jkzI;I!9~Hj zC1d(=w*=YiigyShrl@KIJ^GP<$a-j_1*U->_nGTw6nl&}!y|P^(lA$G)J0fIjG$mx zHsS>;x8$0j7P9anM307ihZR|1oKrZRWndE}JEDD?2Mvv0*n@d+gouKnfH`x?lfW~8 zp1LZfh6I%px#YGak@)v`M`B=;6|b@E(#lVMuRP(OJ^EuHt8yy`T+6(Cn!~ZYKLJqP zFjQQmEP5TVMf?EGFQm|d0|Ff#l2fjSMT7i}zUq2WGhQm%+s!GOA*zzjr}Qd+=7@kC zki#OG048a@C>N-h$1c&6L{ z*D98RQTPY4s>Nav)BgdM{@btr3iU8dYPx^Gs1%x7LG2%CmqJiTFP1{c0+mIOP@mBO z4>7&c|Fdp1kod3e9*dklCiV|(V-nD%g#H8l)EsIQP}Cgl5nnJl6vpVt-4O_=Ez^_L zB@X@zhf=^-8wkOKWdVvI{71cy8hKsxE4 z_!Dc{m^e0n3ifkUceqZ-almG1ApHObG|gH<(+1r?VAPryt;-0R z=2;d&M7>V8bAaiT?(!dXqk)8fb@y1WbX(bf;1KhiZZ7d3n5JI!-EO5|i~TD?_t$=D zmPoD!mRGmq3-XQ+eZb@uCg0T69VmQHbkI8SFZ3jT52_oV33M#6MKEz&h_s{t6Mz2` zMm8i3<&=!A*m4AjWQQM`y9yN$5Vu{ia?i)EC#>Hs-);PsZ?O7Jq_VQVcDz=(_Pq8` zYu;;-9Vg?a%nmS9mTj=D^KKM3bBXSP)o(*ub#05Am>22GL$syOHA(8G@y?P%EJrg# zopwIH|JyRPp&|=2#YTX@4gAfDyuvGRS8oy+kv~-4n4ROaa`$z7K80^!Ui$uN7Tq6l z*BQeL;2@)ym>ixQn;c~(2?BRmet5x0>5VTW3MIsc0+KQf87& zCaFIp@{$_}pj&bL}E&P`wAhabP>vtXL5 z>J(`-Xqr}#R*{w^>W168!Esu4RE-Q0$D!(%!)TdC?{azuwN{$BT~XySLiJ z!6wTz#d#Q+h(e&PUcUF`4>v6jP2YX?WCv`0S7!Ri-fg@1IG2mPij& zd*f@07qJqGp4nKn9g}9z$e9U`Y^FDpowkUbTz$5t{vc@Z*eSsqw>@hJZ%i<%+b8q@ zPyYt=7)Am%-?^P9VO=eNAsIXyMBF2Ru`2NWQSk&dVz42oV|M`~Flkkg(bRZ-%{rSu zlY6tQ_vBC?z@t+roRp+}WbSIJ(IE-7Hvxv6bfra^1tRN88BUH!;}ecHg=`9sd^W-C zmW6JDfdmUQB#|AWN|`C!J=s0gBPudSfTwS)xvYG^y)AUi{3c`?knK$mBYU1zPa z@``pz1t$YfGf?zmTM+|HDvRe?e)Ga%{Ocp*@IzMS^sNKg$P4`-$%osHxP%i)GmQcj~t4cy4#j&rrYdoq&-!Z$)P{rAI#`Gu^tEJlTO>(xS}3p$ksX-2AEU z{EM7Cy`mk>x=AlL&lOSk66mKw!_Tf4++N9myPw+@_uQ2&&yfZ=7@Nxz(#h%fV#v{> z02!qrBy68vaQNt-x3KT*O1&doB&Se+79mvow#LjTBx(7fXdWY_Qh{s`l9x6@S&q@} zAX3)T1t1i{Fp@s>R0x8UR3UbI&ibM8kuXG7-#Iw>fuL!<6KBmiQg<8g*EZHVth*WJ z^~@*QeDp-goClNHJ_Yzbq%|$fkC>&OwnR%&z55ElOi#@Zc*KkFla0MbW6=SZH=Cq# zQF+6j^T6{37osz22Iq>U$k8G`+cNr1w5}oviuC?LF!V{fM91)CAL7l9OnaUbVguki zDJcZn+``BItXtwlE(mK0VWu#iWkly>H4O@qOcme1`)`<%OvtchDlB~BP2*C})zuG) zcR|p){Z2k8hm)HdyV|=JAKBuCF5fosz@Oic=TZXqed#qj{;Z>H!gZ^LlHL(S;pRl_ zO@;RM-GBuG(EgSxb}R0D+(D)6RWgrN!kpVBo0aQ`V8Pf7FX3Br`_+oW6G|l91b06W z)bbO94|X&*MJjD_oY;-%32$+M!B4DRb(y}474yoq#an23R)!f&cecAVPXfTaFA`T6 z6Po|n?0QJE_@&%iAsSD3#k#A1m7jjZf53lQu>Xd^`b*sf#t2Svt`+U(lKverdfOBX z^(uvINmaAco5?5BQ(-_TOS>S`0nsp`ikKF$7I8T2;F`~88esS;*CB6E=2a~$oG0w< zQ+V687dY~QI--JpUj#30J4eLEKpJ_tEqwMS`G_xKaTWfSMBY5c&8NVrFMUC!nws^o zgJqvj4fZA`kwkL&1-lyIWuS6d?7e#gw>*k}OZm+H7WGaiCy~!4p$8D6Yma2vDF&m> z0cheRhk?76u9-*DGnNWj3Jc}T^E}5e$LS2Z+7+L*0xLdpv6uLs8eRWT722-3Y8NiR z?MoLav6flfmzZd}TZ`bRF`xLZ@|y2=I3 zLrV0RCLdHjly=z#Jy(%R8R|(nu~=Ecwh>9fFF2x%p)x_cTrYdQR21j4I-zF)p1 z(qHOf$Hz?uV)RX{9_$`WmC_}pT|lFDmr9pxfYLgB52Z2r;?4{CF@KGrpX^DBpOzL* zMVxv?-9b%jwTNcjFUnukCyOT!i=wuK^2qWsKmWX&VlT)Ug@+&Me8$d7P1D1O*NrpF z_eFdL5`K##1E_+FO~dpAPjFQzhuLUqn&O@AL!noYR`#v*YD*ytpNA}F#S`Z(cH?g` z`J@=+${(}USi8l)f>4_01Uix~sQ1ihe^ziF%Rr2|u3slwbeSS~r#_Vvw5Ft4a{VaW zPp$xqB%oKO;3S_0#>7b}(j;q!5cZ*%;~3qHZq=lzyI`yMkfK8ki&_MogrOYG(Qa#A z^W~GLJ{V3pL0VJ7S?A=jZ#qL~-c7g`Fc071#}bt5cz*I~+ES@;*RmKs-Nt@Jq%9Z5 zSKnhx3p#dr2u_?8}NtKaLe%?;ep+z#o}yC zQ8GfKt%#f#u&Zm)^waS(^Fe z1rAD9GEB8J+w>lBMA~@rc}|?bqsjNbaG<-tsMS}HA*U*j$6Mi#acElOf8A%daZHI2 zi}UuO-s(l*EoT?t#JNS_3lYum55tooLuUIw?WljkC|lW@`~i=*LtzvV?}EWHAl?3d zA+M-v_jQg4TYwP4k@NDQG7axJA%A?iQrS-h2Dj)jzpAOwaPCv2W-)o$!}*87nioIT z<)apUV9orIAf1mQ4M3g*Md?X8JOTg`NlYFO%PXSosl!h9M2UjnE9lsvM$EGv52__Zl=DzmIVSTo|ALurE?Q5L2ju^~r=Od0-8==rIvhWnk!9XP4^qpf}C zFKf~YzbdgdZT}2sV^eeX*f$UKkL5;K1G7V?4qMuRSd2w6-O7dDgtBD|B))5v< z!1owQSIa0u7w9n7evcR{=7Fr%Hc(He&QMR)A}kPQ@TyNl!9d5!t7ujiSBpKXlD`LJ zo_$0$im|NlW3C$15e6vQ4@K`f3+1UQ3^Gz{>T&Fo5u(uh+^C z<9OkQS$0dUYodna*g_RMRaK&l84^j+4H&>TF-7~5@xrPUsSHca$z0^0?zyVm<{j1r zQJ=X0F+2*@xLhYfidkKj>EvhqGfrYFEhrR)E8%`kLlb@PqoZ z?kvk-U1$UTyY=HdI@M6MrCH*4se~r8OFn8$jI_%6sMTV`9kj;erghlxh&Kl`Vn0z-Q8RCR^rO!(#W(Cq&C0HKDw+AYxsmeY2tl2)P%DKv$ zF3yESY(OY{_e)| z!hckL@eE;fYpkaGy(-vu+HYEAY&F~j7;JHkck99?HAvB^OnuIC!Gn~gVEOGfR7z6P zAyix(-O-E`6;@el-isTkSg%PG5+A?mkOcbJDA5_>8DWXfFh}WsDCKuz2qEe z)YM(ytFwi6;Q8u4!}wb=*gY7_6s(0@kY=iZRXrHWYv!~ZeS-kU_-bw{!r&tT&N%Uf zxVbg_o&@uK^b%2^ezAs&p+Fw1ga|_mTMN7XH+~EG2*3g~$3|f*1+M7-Djtt=c?|h& z(S+y+n^%UH_P%l>gJXnx9U5vb?k~a?ilgb`=(*z+1f26T0ER;{rWSPHLL_8VGkfICBe|g z_O&wi++QEc!*ovSqWtXs>*7ea%q0GIomS`$M(tP()nYQZX-xW-rbGX(YP%7 ze4we)t@0{d;j5F_O;YTXilp3p=_Up?+Vxa%cmk0jfi)=+?v(M$l}gw&vpC|53EfQZ zg`u=6ynH4bd!b?7=P<`OTJ~e}Ws7$ee&fPxm9{U>;zw2gwp*m-LyF4)8W)}6_=ear>w3% zTB~m#Z{grJl|H~;=q<_lQ~rmAwKdHF6<%rq|6^FQ$l4) zn_BA=Yndsotl`&s9jkZ?$k7NZS-iO6&|zTYoOkWz+{vvtG#ak@S(|$5>KGFVE;>50 zK$N^zPU|g>OW_aAB-XIg!f1O!nD0(S8iYo; z2Ds{2XNaECM$RV0zIfe=f8%y=ZnlugvQ?{9Y*iu(;QvbF@NhU-KK?1mlEu!Myk+V3 z^RQKsO4Qe*Vn#Hf-@xt?a8o?8!043pmbMM|d6l+dfAy$lXSDSJ)Ws5~54*a^>1>ly z?2cCtlM-Tn^0I^xyzU2k;z#L$)tTm{# zBk2pyMc@EQ3Y8w0{GdF3$7e^w2j{L@^0yT{ISP!}++|V=*R-ToMTs>6@+hG$B4m7C zX^66mxlN+>evq`EQk!lkFNjg0j6HSosG^o)V_)15(!0h>y1Uk2`ETe2=pFBFCNz}2 zpMB^4Xu00SUrkQO<<#3Ygytd_A35$Tb%(1Hs2@A~b}lyS|If zf1kOtQdUnh6Unxwe$SozjL;1dV?vPJr<<9RmyNjytg|J4L3)aKRcQH4FQHIeLC#H% zNzOHek4Z>_Pmf?t8iVu8pkNyRTl^>ZS?bR%Ui{?totZx0rHHJo-{WHx#|&(oco$@D z=sDk}zqDamm`m|_Gheqxr)lm`3C*WxolY_;(@*SR)fA;;ie`XN?)R(J$!QAHxS41z z6lj=gkV|*6jhh}Z)!9=W48LzRy1LaLD*8@POWzF&x7!yA_8d2jXtx!1CZPH|=y>&? zUL zuU|5}HTY%q;o8>;FOZ-cr*5nU($%pHG-i|Ma~~!y3mi(q3tbYK^yhXOV`7QC6C3oe zXb&E0vY{@Ab6|H--Ll@A8pg76iw1*G@k}dBcjg2%;(pC@3{HV^Fsu$bk2Y#MxxJ4& zTcVFJj;rsJ-xZG)7j;9UM!B5l+q76eo6ISG&0Va;AtaADuk84Q#hIe6MI>8O#!yOB zX6KQN16v-jHR7A98ts^i`ZW%z`a>W3G#)PL5V(Cx?r*UQP|uXX`WTje~5{ z7EzdYhhNcRYFv4R_~;jcOx58i49(hD{B$1h7e2A%y*!WwqFZCfj$6e6M0fx8vGDBN|;x&-&V~FA*RV7W%oi!5na|bklRIC{SXWvbo5~ zky7HJ>%}d0o8Qwy`|bn>b@aT@fY9~0MLRDh^i$(h$H_aV^uTm9ukik6(3U3C!dGZ? zw^>NoWplECEU6Z8%Umty?+OxrwsvaDY8KVi3$UlmetvbEbydF4<9@Z5rSWHyzdnC) z3$67C7q_3awk7&r%JbRos8UTp(8KQN{^OFrc{K!PY~KNy*fGa^qHNt*a@PhG>?>U5 zX7PKCklQYv*}=*-)yQMDk|~Gbr(vfBuKUS1{R%;k97=ZRdZ8wESf$UTC*`-N^A?`|*clU5wXvF;fjH)z0Iy^jFhqZfF z^Yht0A$|}K)lS(2r-$Nc6MQ!=xaTzj$3oeb>%_D55z(bdP#b6IPhhjV3x^400RH5t z030>wx=}?jtbv$dc=Fzs>Hsbk!&TbV8*Y=}L1X>JmBBt=L=C%4FVUY+6`L);fLTD< z&R1$Bz?XkIa~LGdgQ$v46}~KwfWbJ%;T8uUlAC21L-(hQv268GC^w(m z4uKc+B4J;*91T+TI{RfgGd<4HD40C%L==X0#=C5}vxG9oMLChE&&n5xst^Vnh__Im6u`_Wag1(z7!?HZ}Rx4FtEH~8N zmamZL$yXt7l>Or&+QEa4GE|)W$=RQ^-da*vCx{A z%#@w#7ncHmZ}}q&9}-yGWEzbVVRPwxHDtDHyZ=!PEufRpC4IA+YH+jqB? znvCS{(OZ9Ek)(ANkL4J9?EX{J{K8hCPhxqMQUN<3jWH^36n0X9uj*Ql;csLj!D&nk zQ97Z@=j+`(r{E=&vn0jl}N*d-JMQW6heHGPl(2l14I6koQ);Flq6 zg@hp{+eLxzn^VLj$dz7cYgAf*laWqyJPbiF{LjQ}^l1@O$40ngsFvdnj}Q+IB3&-G zWw^XZDb1rVV4*#lJ+Cq~ppmtajgYO7ReGN-+r$WRSxoec{n}gGw7ECxw`2C#LD8}{ ze@p!fse6L6<7=bAW4TONmFpR+lshCLk|<=f5xhw_Dj|`fF~`s5GJ|W@UUC8VU;st%byV_nVZ3i?&7IvoR`T zW82G|Tk^gk=MXiCW46@FX|w!9p|2b!H`U-4os5iuz$ixICG<4V&Sqy9BG%(Tiraqi zgT6A~e(&%3ws`AwMChg<$<-TNe&Yy#cz*cjw?k0Yh`d4I0x5e~%0T(ti}6jHi8uwb z<_(Lz0!>Pu_0$jRQ`prDb~5+Gob_jz;}J>B;Geh+P+O)La9}WP8M}L0Bcif6Wgz5x zFOSK@yQzFWZa-aj-7IEyMFX}0jCF>7$s#zFL9&E9x!R<1*1#5Ny>wRK3PyyZ49ZMJ zty1E&MBqpz5>@M5t3c-n(EbtA_Oj^Mh)@XY_3w-<{iAGTB_QhgZ&4Ctc~qjBZC7B_ z0euVf^Xw#!LF6rQq|RXetY74BF+IHq3HuT@3jGq#Ul;*$B~WI1QsEM(Kp&}22{^=N z2{Z-|bmRn|q&H75b+L^wrBFm>iS$^P+&3~h&q}IDT=D_zpLRIE>y)zSTHs46f%!U| zz1cL3T+ahsg$soBOL+`92GGsN3M%$Ef1hoOf&*J-dPP$?{zg@ z*LkkrU-$F9pYQ$L&-3r^^RRcNOEJT85Vl#S58DkkPlF;BR3cADkYo!K5t}-ZKI_Ru zWGdr&wR0ADTN?=>-?~I8d4KLmX1PZCwMt{i1|LDK_pEQLwu7<6F;OS@TA!-#P7X423`0TOk>zcA`^S_hpe0Vr3x(_=xjrgZ93}( zo$oYUTE{5)>PX(c6Pr5{T_duzi_9q}@C(i~ws?0_bJ#-na$k3zQXM+T1V1Hr_ zq-4B8V?pL+Qa_!-bdy+V9B?m(d?fR!rFmV0-CZT8oG}LTl|2D*@J~q1Sx)K_x*5H8 ztQ{7%zsgY&qZD-&JA5RqS!-lk>hgVMZ%O){a7Vx4`G^=n4QyVf=rf|Ly#M{sXm^L$ zYu#8ZOSOdhG)~QGjP!fUSd@5L|k+yucT`4I(WUG6ja5dr;IELXkE!PC5u z&5f5hnfF7-W;VYKqzP#E4G*jY=qi+Jv0o6QXdC#jiO-zkYk}Moef`*LLq$0IksPho zUC~*FsWzDKbDv0?r4#EDdh|L(BIgM~x0dFuw1N|a8eBxO2g&-hZ_uwCtHejB1M{Tb zV_&X4T3Z=hS(!j5SdP60`=gduR?vpuOS-Qh%SOo?!SZYR<8XFQ_hurU?6@(#LyQd+ zgA!Gbj{^tP1p3e1l#k#!*QXyT9+e>aV;|2UB+Y$8CYp?!{t?X1*uoIUZlg;zB&kQT zZVXTDI({try5H+e(D@qfSJD~9*yPw*({=sFv$JBTa3ywlU6mE6=z2NJv?gGr7eLF2j8dy1%Mb6+t^;D6uYFa7XN8_sYQdhXvx}A!>r>^tg3q)_ z?2hr`z}l6zahs8oR+7_W>vi%Tdkn=^L#Ug_b5{F1*-^~F+5)Z1fsQ9tsq>%b7iA0d zDza*`wib`h0W4ww7O_B-NzH&08ey;L{{p@NutW0CW%OA0x8PF8b}p&F{JILGb;2)5 zB`)CcV^7Pfp&IS;_E;LtH;aY~Hz7ySVuLzgI9(2Yvb*P1(U>ZhhQCZMN*@{}cC{on zjrn_cxtS!`mHC#Ex?ZKCM_Xjg5^|nz-I|JaO>8Hc$K<#-(iudP%BiBKT-m z%h9q^*_FO?)|9kdk1SgA4G!Lyh;-pSyT8+(KC&;Z?|eI;yHFXG^=Bm=c&1EE3d6mA z>m{$)B@Ml5pN0l7*iyM4;|8zV$3c@hu&t-IurU%+ z640ZYZ7MrjT^4D{s;`-Pi0_aiQl&4UW{XY*n|b9(w>x1b-x-CE2ZOzuS@;0$Al3(l zHmmd?)wT6|A*q|VKvxOiltiQM?yFbteoTkki{;B~6Z>mWxxpKm}W92E1sxKfic?ew4?{a)n|q(MUFc z=R$aJ==a;%J3lWJSfvPmf^q#F1NI0X&h?EXZsgfXNE`=RrXDVo=&~p`!meIwmuo;a zP7c3DzdRDQ^rn&vs%q|VqNT(&yv=mRu{=WqOwjv^Z=yZ5tHVmEiIq~4Ns#lr>>l&TuV())R2+yz9=xLN%?$e z9Ep(F7441)RB2pK?*UH9cKg)_^%9KFv`6HD)~{N0sI__8yMHOX6x8r(Pz zRd(S-d~L6xM0&!Aoa;WR>go>X$)aF~jo@)@mx>IgN0VI5;(gch=FGY=v=USut zB>-~w`n`;UhDM!O==VIo@rLvCl0e;VHONKV#qqXwD4?{`L8e&3jb&Q<`%2m*#9%R3 zdf~JK#&e3QSj6B2c$;U&^}UmO(9?ZjCaHKp?a2j+JViNXhjuEsEhR~-p!t@W&zNL9 z=}yeK)~1%G_c-i6FYiVnfy;Td?O=xDM0C^PM156j6sNNjj?^c2WSg4s*JI253i)-) z=|rOhuRXM_nnw2GS5rTLwxhoM-sS>G8L{g-c8Tclr`){GcFdvnm)MxLb!sgK@4~}z z_%1y8AlBla<9$*|$L~CE*|a_%PXpqqo9?$0JNKU?&TKi7d~W23zRsLliq{|J5-%S8 zF)(f=DST?3*|-5UZDCpnc()ziMhq(rBf@81e!5sT6dkhx1{e)G+3#S|H)sgY9HMzzzcsgKAPEFpR{*O@t$LZzuzZR!Vo9o`nGISIknh5 z%qYco@!4*#PpJdV01wk6jL<;HxjP<$dY2)ypoQ(RM#i5YM-Rujh?Vv{2@muWy{DJu+DCTMXYeJ1L8o#fTqx zTPJ&!`CMft2~pqs!Nb#m5xTsGM*1p&nO+LKeKM&g6foE;jUhwbMrWo5c152TV>`Q% z047t5ov*kNzhz|kedMLUJej#={=G!FP>01&HCs@@N>D8v9?ylmtIc~NR z&IHMIU75A3++BfWM@=AYY>p-V)0bxS2oe9g>cbZKPRG6S_hc7e2PUJ+`!6!YDAUc} z<{#$I_TDUMH51p}H{fCYfd89JZoE>nQaedYQ~Dv+!!@KN|r42l2=**kyDDqQ|w46Y4^ z{u~p+XdpsjkBANd))mo#pl}_C=z1`SkQpSJ3)h8;>fkV4knS%!xUP`x^JDy7A3_%b z74a3;2L;1_)`7LPVM3zLPq{)JLR-Wa41$88qQ1f`(H{I*NLM5m1VZRSMfV0mz;s|g z`wDe%A=l`qT0mf-2SOwU0zsi7H4AlMT~QBhunt_bFWO+Zj_7(iFp#$BdO9#2A>!%h zTA(nvwrCH)Al+Yj4+g=1_0@(#ME5J~{V%yXFoX~h^|P-I94rJ&{T&M=dI%X!L~$YF r%FZ{;`^VMQvLz4#6@DDXe>hyEKo4TzUpE`94H2HPxRR2o)#-l&X5~%^ literal 0 HcmV?d00001 From 9ef00d187f3b96312488166d1cb9edbaa0df2c6d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:58 -0800 Subject: [PATCH 276/374] %s/master/manager/g + better comments --- uncore/src/main/scala/cache.scala | 20 +- uncore/src/main/scala/coherence.scala | 255 +++++++++++++------------- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 2 +- uncore/src/main/scala/tilelink.scala | 142 ++++++++------ uncore/src/main/scala/uncore.scala | 12 +- 6 files changed, 226 insertions(+), 209 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d871e652..0e963f24 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -118,7 +118,7 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { } object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { + def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh @@ -126,7 +126,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata + val coh = new ManagerMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -156,7 +156,7 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -322,7 +322,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire finished transaction acks val finish = io.inner.finish - val finish_idx = finish.bits.payload.master_xact_id + val finish_idx = finish.bits.payload.manager_xact_id trackerList.zipWithIndex.map { case (t, i) => t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) } @@ -355,7 +355,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { val addr = UInt(width = tlAddrBits) - val coh = new MasterMetadata + val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -385,7 +385,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val state = Reg(init=s_idle) val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) - val xact_coh = Reg{ new MasterMetadata } + val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_id = Reg{ UInt() } @@ -406,7 +406,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) @@ -701,8 +701,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (xact.addr === c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c7999f4b..08d7cf20 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,6 +3,8 @@ package uncore import Chisel._ + +// Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters class ClientMetadata extends CoherenceMetadata { @@ -18,21 +20,23 @@ object ClientMetadata { } } -class MasterMetadata extends CoherenceMetadata { +class ManagerMetadata extends CoherenceMetadata { val state = UInt(width = co.masterStateWidth) val sharers = UInt(width = co.dir.width) - override def clone = new MasterMetadata().asInstanceOf[this.type] + override def clone = new ManagerMetadata().asInstanceOf[this.type] } -object MasterMetadata { +object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new MasterMetadata + val m = new ManagerMetadata m.state := state m.sharers := sharers m } - def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) + def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } +// This class encapsulates transformations on different directory information +// storage formats abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt def push(prev: UInt, id: UInt): UInt @@ -66,15 +70,16 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } +// Coherence policy inferface for clients and managers abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int - def nMasterStates: Int + def nManagerStates: Int def nAcquireTypes: Int def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nMasterStates) + def masterStateWidth = log2Up(nManagerStates) def acquireTypeWidth = log2Up(nAcquireTypes) def probeTypeWidth = log2Up(nProbeTypes) def releaseTypeWidth = log2Up(nReleaseTypes) @@ -82,49 +87,48 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def isHit (cmd: UInt, m: ClientMetadata): Bool def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: MasterMetadata): Bool - def isValid (m: MasterMetadata): Bool + def isHit (incoming: Acquire, m: ManagerMetadata): Bool + def isValid (m: ManagerMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: MasterMetadata): Bool + def needsWriteback(m: ManagerMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def masterMetadataOnFlush: MasterMetadata - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata + def managerMetadataOnFlush: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: ManagerMetadata): UInt def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt + def getGrantType(a: Acquire, m: ManagerMetadata): UInt + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt - def messageHasData(rel: SourcedMessage): Bool + def messageHasData(rel: TileLinkChannel): Bool def messageUpdatesDataArray(g: Grant): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: MasterMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool + def requiresProbes(a: Acquire, m: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { MuxLookup(a.a_type, Grant.uncachedRead, Array( Acquire.uncachedRead -> Grant.uncachedRead, Acquire.uncachedWrite -> Grant.uncachedWrite, @@ -135,14 +139,14 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -154,8 +158,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) - def isValid (m: MasterMetadata): Bool = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) + def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -167,7 +171,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -183,18 +187,18 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> clientInvalid, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -217,7 +221,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -230,12 +234,12 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -245,29 +249,28 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -279,8 +282,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -295,7 +298,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -316,18 +319,18 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientExclusiveClean, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -356,7 +359,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -369,12 +372,12 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -384,31 +387,29 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -425,8 +426,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -441,7 +442,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -461,18 +462,18 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -501,7 +502,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -514,15 +515,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -536,34 +537,32 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -580,8 +579,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -596,7 +595,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -621,18 +620,18 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -661,7 +660,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -674,15 +673,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -696,34 +695,32 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -739,8 +736,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -755,7 +752,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -792,9 +789,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -803,9 +800,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d r.is(releaseInvalidateAckMigratory) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -837,7 +834,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -850,7 +847,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), @@ -858,9 +855,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -875,19 +872,17 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 378ca3b1..4db133d6 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -134,7 +134,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } @@ -201,7 +201,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index a34e91c1..161a9a0c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -case object LNMasters extends Field[Int] +case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 4e5ac961..af10d455 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,10 +4,13 @@ package uncore import Chisel._ import scala.math.max -case object TLId extends Field[String] +// Parameters exposed to the top-level design, set based on +// external requirements or design space exploration +// +case object TLId extends Field[String] // Unique name per network case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] -case object TLMasterXactIdBits extends Field[Int] +case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] @@ -15,7 +18,7 @@ case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) - val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits @@ -25,13 +28,23 @@ abstract trait TileLinkParameters extends UsesParameters { val tlSubblockUnionBits = max(tlWriteMaskBits, (tlSubblockAddrBits + tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) } abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters +// Directionality of message channel +// Used to hook up logical network ports to physical network ports +trait TileLinkChannel extends TLBundle +trait ClientToManagerChannel extends TileLinkChannel +trait ManagerToClientChannel extends TileLinkChannel +trait ClientToClientChannel extends TileLinkChannel // Unused for now + +// Common signals that are used in multiple channels. +// These traits are useful for type parameterization. +// trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) } @@ -40,19 +53,17 @@ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasMasterTransactionId extends TLBundle { - val master_xact_id = Bits(width = tlMasterXactIdBits) +trait HasManagerTransactionId extends TLBundle { + val manager_xact_id = Bits(width = tlManagerXactIdBits) } trait HasTileLinkData extends TLBundle { val data = UInt(width = tlDataBits) } -trait SourcedMessage extends TLBundle -trait ClientSourcedMessage extends SourcedMessage -trait MasterSourcedMessage extends SourcedMessage +// Actual TileLink channel bundle definitions -class Acquire extends ClientSourcedMessage +class Acquire extends ClientToManagerChannel with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { @@ -61,16 +72,17 @@ class Acquire extends ClientSourcedMessage val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) - def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) + def allocate(dummy: Int = 0) = subblock(0) + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) + def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) def is(t: UInt) = a_type === t } object Acquire { val nUncachedAcquireTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) @@ -102,29 +114,29 @@ object Acquire { } object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz) + acq.subblock := Cat(subblock_addr, operand_sz, alloc) acq } def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB) + apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) } def apply(addr: UInt): Acquire = { - apply(addr, UInt(0), UInt(0), MT_CB) + apply(addr, UInt(0)) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := write_mask + acq.subblock := Cat(write_mask, alloc) acq } def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), data) + apply(addr, client_xact_id, SInt(-1), Bool(true), data) } def apply(addr: UInt, data: UInt): Acquire = { apply(addr, UInt(0), data) @@ -136,11 +148,17 @@ object UncachedAtomic { subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) acq } } +class Probe extends ManagerToClientChannel + with HasPhysicalAddress { + val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t +} + object Probe { def apply(p_type: UInt, addr: UInt) = { val prb = new Probe @@ -150,10 +168,12 @@ object Probe { } } -class Probe extends MasterSourcedMessage - with HasPhysicalAddress { - val p_type = UInt(width = co.probeTypeWidth) - def is(t: UInt) = p_type === t +class Release extends ClientToManagerChannel + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val r_type = UInt(width = co.releaseTypeWidth) + def is(t: UInt) = r_type === t } object Release { @@ -173,18 +193,10 @@ object Release { } } -class Release extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) - def is(t: UInt) = r_type === t -} - -class Grant extends MasterSourcedMessage +class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId - with HasMasterTransactionId { + with HasManagerTransactionId { val uncached = Bool() val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t @@ -192,29 +204,30 @@ class Grant extends MasterSourcedMessage object Grant { val nUncachedGrantTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id + gnt.manager_xact_id := manager_xact_id gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) } } -class Finish extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId +// Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -226,6 +239,8 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +// Converts UncachedTileLinkIO to regular TileLinkIO by pinning +// probe.ready and release.valid low class TileLinkIOWrapper extends TLModule { val io = new Bundle { val in = new UncachedTileLinkIO().flip @@ -245,13 +260,16 @@ object TileLinkIOWrapper { } } +// Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { - type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId - type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + // These are filled in depending on whether the arbiter mucks with the + // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits - def masterSourcedClientXactId(in: MasterSourcedWithId): Bits - def arbIdx(in: MasterSourcedWithId): UInt + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], @@ -267,7 +285,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { }} } - def hookupMasterSource[M <: MasterSourcedWithId] + def hookupManagerSource[M <: ManagerSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) @@ -278,7 +296,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { out.ready := ins(i).ready } ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) } } } @@ -291,7 +309,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int) } hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) io.out.finish <> finish_arb.io.out @@ -306,7 +324,7 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) io.in.map{ _.probe.valid := io.out.probe.valid } io.in.map{ _.probe.bits := io.out.probe.bits } @@ -317,35 +335,39 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } +// Appends the port index of the arbiter to the client_xact_id abstract trait AppendsArbiterId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } +// Uses the client_xact_id as is (assumes it has been set to port index) abstract trait PassesId { - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = in.client_xact_id - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Overwrites some default client_xact_id with the port idx abstract trait UsesNewId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = UInt(id, log2Up(arbN)) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = UInt(0) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Mix-in id generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e8e82d22..d2531a9e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -194,7 +194,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), + co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId)) @@ -261,7 +261,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -290,13 +290,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.masterMetadataOnFlush), + co.getGrantType(xact, co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -316,7 +316,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) when( io.inner.acquire.valid ) { xact_uncached := c_acq.payload.uncached xact_a_type := c_acq.payload.a_type @@ -389,7 +389,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } From f58f8bf3859ec09ddff6c33b488c5c23bbdd850a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 25 Jan 2015 15:37:04 -0800 Subject: [PATCH 277/374] Make L2 data array use a single Mem --- uncore/src/main/scala/cache.scala | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0e963f24..af3a53c7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,24 +214,22 @@ class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataW class L2DataArray extends L2HellaCacheModule { val io = new L2DataRWIO().flip - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr val wmask = FillInterleaved(8, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) - val reg_raddr = Reg(UInt()) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { - reg_raddr := raddr - } - array(reg_raddr) + val reg_raddr = Reg(UInt()) + val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + + when (io.write.bits.way_en.orR && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en.orR && io.read.valid) { + reg_raddr := raddr } + io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - - io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? + io.resp.bits.data := array(reg_raddr) + io.read.ready := !io.write.valid io.write.ready := Bool(true) } From 973eb43128e1294030f0527e530e0f71150d6023 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Jan 2015 19:48:49 -0800 Subject: [PATCH 278/374] state machine bug on uncached write hits --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index af3a53c7..468b6587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -870,7 +870,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_resp_done) { state := s_meta_write } + when(local_data_resp_done) { + state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + } } is(s_data_write) { io.data.write.valid := Bool(true) From 7b4e9dd137cd907760c82ce984de7d37a67ec939 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Jan 2015 20:30:52 -0800 Subject: [PATCH 279/374] Block L2 transactions on the same set from proceeding in parallel --- uncore/src/main/scala/cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 468b6587..e7f0ce8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -688,8 +688,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data io.has_acquire_match := co.messageHasData(xact) && @@ -857,9 +857,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read) { - io.data.read.valid := Bool(true) + io.data.read.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) when(io.data.resp.valid) { - //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } From 3aa030f9605e0dbd87be440cd747e98d0320470b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 1 Feb 2015 19:57:53 -0800 Subject: [PATCH 280/374] Support for uncached sub-block reads and writes, major TileLink and CoherencePolicy refactor. --- uncore/src/main/scala/cache.scala | 536 ++++++++---- uncore/src/main/scala/coherence.scala | 1165 ++++++++++++------------- uncore/src/main/scala/consts.scala | 6 +- uncore/src/main/scala/htif.scala | 26 +- uncore/src/main/scala/memserdes.scala | 14 +- uncore/src/main/scala/tilelink.scala | 407 ++++++--- uncore/src/main/scala/uncore.scala | 197 +++-- uncore/src/main/scala/util.scala | 6 +- 8 files changed, 1343 insertions(+), 1014 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e7f0ce8a..6651056f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,32 +2,26 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] -case object WordBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +case object AmoAluOperandBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { - val paddrBits = params(PAddrBits) - val vaddrBits = params(VAddrBits) - val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) val blockOffBits = params(BlockOffBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits - val tagBits = paddrBits - untagBits + val tagBits = params(PAddrBits) - untagBits val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 - val wordBits = params(WordBits) - val wordBytes = wordBits/8 - val wordOffBits = log2Up(wordBytes) val rowBits = params(RowBits) - val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) } @@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters { abstract class CacheBundle extends Bundle with CacheParameters abstract class CacheModule extends Module with CacheParameters +class StoreGen(typ: Bits, addr: Bits, dat: Bits) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + lazy val wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { + val t = new StoreGen(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class AMOALU extends CacheModule { + val operandBits = params(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = new StoreGen(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.write.ready := !rst } -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters { +abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 val refillCyclesPerBeat = params(TLDataBits)/rowBits val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + require(refillCyclesPerBeat == 1) } -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { + def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { + val (cnt, cnt_done) = + Counter(inc && data.hasMultibeatData(), tlDataBeats) + val done = (inc && !data.hasMultibeatData()) || cnt_done + (cnt, done) + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload)._2 + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { + connectDataBeatCounter(in.fire(), in.bits) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { + connectDataBeatCounter(in.valid, in.bits)._2 + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } +trait HasL2Data extends HasTileLinkData { + def hasData(dummy: Int = 0) = Bool(true) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) +} + object L2Metadata { def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata @@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { +class L2DataReadReq extends L2HellaCacheBundle + with HasCacheBlockAddress + with HasTileLinkBeatId + with HasL2Id { val way_en = Bits(width = nWays) - val addr = Bits(width = tlAddrBits) } -class L2DataWriteReq extends L2DataReadReq { +class L2DataWriteReq extends L2DataReadReq + with HasL2Data { val wmask = Bits(width = tlWriteMaskBits) - val data = Bits(width = tlDataBits) } -class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { - val data = Bits(width = tlDataBits) -} +class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data trait HasL2DataReadIO extends L2HellaCacheBundle { val read = Decoupled(new L2DataReadReq) @@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule { io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) io.resp.bits.data := array(reg_raddr) io.read.ready := !io.write.valid io.write.ready := Bool(true) @@ -261,10 +353,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { + def doOutputArbitration[T <: Data]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } @@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + doOutputArbitration( + io.inner.grant, + trackerList.map(_.io.inner.grant), + tlDataBeats, + (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer @@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr = UInt(width = tlAddrBits) + val addr_block = UInt(width = tlBlockAddrBits) val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } @@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val pending_probes = Reg(init = co.dir.flush) val curr_p_id = co.dir.next(pending_probes) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) + val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) - io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, - UInt(trackerId), - xact_data(outer_data_write_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), { case TLId => outerId }) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.wb.req.ready := Bool(false) @@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str is(s_idle) { io.wb.req.ready := Bool(true) when(io.wb.req.valid) { - xact_addr := io.wb.req.bits.addr + xact_addr_block := io.wb.req.bits.addr_block xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id @@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str when(io.inner.release.valid) { xact_coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - xact_data(crel_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_read_done) { state := s_data_resp } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_write } } is(s_outer_write) { io.outer.acquire.valid := Bool(true) - when(outer_data_write_done) { + when(macq_data_done) { io.wb.resp.valid := Bool(true) state := s_idle } @@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val wb = new L2WritebackIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - - def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { - //TODO apply acq's write mask - Mux(co.messageHasData(acq), old_data, new_data) - } + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cack = io.inner.finish.bits + val mgnt = io.outer.grant.bits } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) - val collect_inner_data = Reg(init=Bool(false)) - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (local_data_cnt, local_data_done) = - Counter(io.data.write.fire(), tlDataBeats) + val collect_crel_data = Reg(init=Bool(false)) + val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) - when(collect_inner_data) { + when(collect_crel_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data + crel_data_valid(crel.payload.addr_beat) := Bool(true) } - when(inner_data_done) { collect_inner_data := Bool(false) } + when(crel_data_done) { collect_crel_data := Bool(false) } } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_addr_beat := crel.payload.addr_beat + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_crel_data := crel.payload.hasMultibeatData() + crel_data_valid := Bits(1) state := s_meta_read } } @@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, - Mux(co.messageHasData(xact), s_data_write, s_meta_write), - s_grant) + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_grant, s_idle)) } } is(s_data_write) { - io.data.write.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (local_data_cnt < inner_data_cnt)) - when(local_data_done) { state := s_meta_write } + io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? + } } is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } @@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) + val collect_cacq_data = Reg(init=Bool(false)) + val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) val crel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) val curr_p_id = co.dir.next(pending_probes) val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(co.requiresSelfProbe(xact), + val mask_self = Mux(xact.requiresSelfProbe(), full_sharers | (UInt(1) << xact_src), full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) val mask_incoherent = mask_self & ~io.tile_incoherent - val collect_cacq_data = Reg(init=Bool(false)) - //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = - Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //TODO: uncached does or does not allocate + //val do_allocate = !xact_uncached || xact.allocate() + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := xact.data(0) // default + + def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + amoalu.io.lhs := old_data + amoalu.io.rhs := new_data + val wmask = FillInterleaved(8, xact.write_mask()) + buffer(incoming.addr_beat) := + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), + wmask & new_data | ~wmask & old_data, old_data)) + } - //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || + xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := co.messageHasData(xact) && - (xact.addr === c_acq.payload.addr) && + io.has_acquire_match := xact.hasMultibeatData() && + (xact.addr_block === cacq.payload.addr_block) && collect_cacq_data - io.has_release_match := !co.isVoluntary(c_rel.payload) && - (xact.addr === c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + (xact.addr_block === crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, - c_gnt.header.dst) + val next_coh_on_rel = co.managerMetadataOnRelease( + incoming = crel.payload, + meta = xact_meta.coh, + src = crel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant( + outgoing = cgnt.payload, + meta = xact_meta.coh, + dst = cgnt.header.dst) - val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) + val outer_write = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), + { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) + io.inner.grant.bits.payload := xact.makeGrant( + manager_xact_id = UInt(trackerId), + meta = xact_meta.coh, + addr_beat = cgnt_data_cnt, + data = xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_write_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_gnt io.wb.req.valid := Bool(false) - io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) @@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(cacq_data_cnt) := c_acq.payload.data + xact_data(cacq.payload.addr_beat) := cacq.payload.data + cacq_data_valid(cacq.payload.addr_beat) := Bool(true) } when(cacq_data_done) { collect_cacq_data := Bool(false) } } @@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_cacq_data := co.messageHasData(c_acq.payload) + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_cacq_data := cacq.payload.hasMultibeatData() state := s_meta_read } } @@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) + val full_block = !xact.uncached || + xact.is(Acquire.uncachedReadBlock) || + xact.is(Acquire.uncachedWriteBlock) + read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) + write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) + resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) when(_needs_probes) { pending_probes := mask_incoherent(nCoherentClients-1,0) release_count := co.dir.count(mask_incoherent) @@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_meta.coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - c_rel.payload.data) + mergeData(xact_data, crel.payload) } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasMultibeatData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -847,37 +999,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { //TODO make sure cacq data is actually present before merging - xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), - io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_meta.coh - when(outer_data_resp_done) { - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read) + mergeData(xact_data, mgnt.payload) + when(mgnt_data_done) { + state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) } } } is(s_data_read) { - io.data.read.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) + io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) + } + when(io.data.read.ready) { + read_data_cnt := read_data_cnt + UInt(1) + when(read_data_cnt === read_data_max) { state := s_data_resp } } - when(local_data_read_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) } - when(local_data_resp_done) { - state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + when(resp_data_cnt === resp_data_max) { + state := Mux(xact.hasData(), s_data_write, s_meta_write) } } is(s_data_write) { io.data.write.valid := Bool(true) - when(local_data_write_done) { - state := s_meta_write + when(io.data.write.ready) { + write_data_cnt := write_data_cnt + UInt(1) + when(write_data_cnt === write_data_max) { + state := s_meta_write + } } } is(s_meta_write) { @@ -886,12 +1040,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + when(io.inner.grant.ready) { + cgnt_data_cnt := cgnt_data_cnt + UInt(1) + when(cgnt_data_cnt === cgnt_data_max) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } } - is(s_busy) { + is(s_ack) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 08d7cf20..6367b44d 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,7 +3,6 @@ package uncore import Chisel._ - // Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters @@ -14,9 +13,9 @@ class ClientMetadata extends CoherenceMetadata { } object ClientMetadata { def apply(state: UInt) = { - val m = new ClientMetadata - m.state := state - m + val meta = new ClientMetadata + meta.state := state + meta } } @@ -27,10 +26,10 @@ class ManagerMetadata extends CoherenceMetadata { } object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new ManagerMetadata - m.state := state - m.sharers := sharers - m + val meta = new ManagerMetadata + meta.state := state + meta.sharers := sharers + meta } def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } @@ -85,56 +84,59 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, m: ClientMetadata): Bool - def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: ManagerMetadata): Bool - def isValid (m: ManagerMetadata): Bool + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + val acquireTypesWithData = Nil // Only built-in Acquire types have data for now + val releaseTypesWithData: Vec[UInt] + val grantTypesWithData: Vec[UInt] + + def isValid(meta: ClientMetadata): Bool + def isValid(meta: ManagerMetadata): Bool + + def isHit(cmd: UInt, meta: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), + clientStatesWithWritePermission.contains(meta.state), + clientStatesWithReadPermission.contains(meta.state)) + } + //TODO: Use outer protocol's clientState instead, remove this function: + def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool - def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: ManagerMetadata): Bool + //TODO: Assumes all cache ctrl ops writeback dirty data, and + // doesn't issue transaction when e.g. downgrading Exclusive to Shared: + def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + clientStatesWithDirtyData.contains(meta.state) + def needsWriteback(meta: ClientMetadata): Bool = + needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata - def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata - def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def managerMetadataOnFlush: ManagerMetadata - def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnFlush: ClientMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: ManagerMetadata): UInt - def getProbeTypeOnVoluntaryWriteback: UInt - def getReleaseTypeOnCacheControl(cmd: UInt): UInt - def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: ManagerMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt + def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnFlush: ManagerMetadata - def messageHasData(rel: TileLinkChannel): Bool - def messageUpdatesDataArray(g: Grant): Bool - def isVoluntary(rel: Release): Bool - def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool - def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool - def requiresAckForGrant(g: Grant): Bool - def requiresAckForRelease(r: Release): Bool + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - - def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -142,123 +144,105 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 - def nReleaseTypes = 5 - def nGrantTypes = 2 + def nReleaseTypes = 4 + def nGrantTypes = 1 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) - def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) - def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientValid), Array( - M_INV -> (m.state === clientValid), - M_CLN -> (m.state === clientValid) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (outstanding.a_type != acquireExclusive) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientValid - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, Acquire.uncachedAtomic -> probeInvalidate )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -266,137 +250,123 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 2 + def nReleaseTypes = 6 + def nGrantTypes = 1 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) - def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata) = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientExclusiveClean - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), Bool(true)))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -404,152 +374,133 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 + def nReleaseTypes = 6 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (isRead(cmd) && outstanding.uncached) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> clientExclusiveDirty, + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), - MuxLookup(a.a_type, probeInvalidate, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -557,157 +508,135 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 4 + def nReleaseTypes = 6 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(outstanding.a_type === acquireExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -715,174 +644,170 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 - def nReleaseTypes = 11 - def nGrantTypes = 5 + def nReleaseTypes = 10 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), - M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( - Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> m.state, - probeDowngrade -> MuxLookup(m.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid)) - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireExclusive -> clientExclusiveDirty, + acquireShared -> clientExclusiveClean)), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireExclusive -> clientMigratoryDirty, + acquireShared -> clientMigratoryClean)))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next, r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next - )) - } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + r.is(releaseInvalidateAckMigratory) -> next)) } + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), - releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData - )) + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(meta.state === clientMigratoryDirty, + releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData)) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + probeInvalidate -> Mux(clientExclusiveClean === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(meta.state != clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck)) + Mux(dirty, with_data, without_data) } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? - ))) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index fe647d07..69cdc41f 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,6 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") - val MT_CB = Bits("b111") // cache block val NUM_XA_OPS = 9 val M_SZ = 5 @@ -37,8 +36,9 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); val M_XA_MAXU = Bits("b01111"); - val M_INV = Bits("b10000"); // write back and invalidate line - val M_CLN = Bits("b10001"); // write back line + val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions + val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4db133d6..5e5053be 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -136,7 +136,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) + mem_needs_ack := io.mem.grant.bits.payload.requiresAck() } io.mem.grant.ready := Bool(true) @@ -184,22 +184,26 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: UInt = null - for (i <- 0 until dataBits/short_request_bits) { - val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) + val n = dataBits/short_request_bits + val mem_req_data = (0 until n).map { i => + val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := + io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } - mem_req_data = Cat(packet_ram(idx), mem_req_data) - } + packet_ram(Cat(cnt, ui)) + }.reverse.reduce(_##_) + val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, mem_req_data), - UncachedRead(init_addr)) + UncachedWriteBlock( + addr_block = init_addr, + addr_beat = cnt, + client_xact_id = UInt(0), + data = mem_req_data), + UncachedReadBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id - io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index d6133fc0..e7291222 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -228,7 +228,9 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { mem_cmd_q.io.enq.valid := Bool(false) mem_data_q.io.enq.valid := Bool(false) - val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are + //acceptable Acquire types + val acq_has_data = io.uncached.acquire.bits.payload.hasData() val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) @@ -250,7 +252,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr + addr_out := io.uncached.acquire.bits.payload.addr_block has_data := acq_has_data tl_done_out := tl_wrap_out mif_done_out := Bool(false) @@ -323,8 +325,12 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), - tl_buf_in(tl_cnt_in)) + io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + g_type = Grant.uncachedReadBlock, + client_xact_id = tag_in, + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af10d455..32ccac87 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,20 +16,22 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { - val tlAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits - val tlSubblockAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlBeatAddrBits = log2Up(tlDataBeats) + val tlByteAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = M_SZ val tlUncachedOperandSizeBits = MT_SZ val tlSubblockUnionBits = max(tlWriteMaskBits, - (tlSubblockAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 + (tlByteAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) + val networkPreservesPointToPointOrdering = false //TODO: check physical network type } abstract class TLBundle extends Bundle with TileLinkParameters @@ -45,8 +47,12 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. // These traits are useful for type parameterization. // -trait HasPhysicalAddress extends TLBundle { - val addr = UInt(width = tlAddrBits) +trait HasCacheBlockAddress extends TLBundle { + val addr_block = UInt(width = tlBlockAddrBits) +} + +trait HasTileLinkBeatId extends TLBundle { + val addr_beat = UInt(width = tlBeatAddrBits) } trait HasClientTransactionId extends TLBundle { @@ -57,55 +63,109 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -trait HasTileLinkData extends TLBundle { +abstract trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool } // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { + // Actual bundle fields val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits - val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + + // Utility funcs for accessing uncached/subblock union + val opSizeOff = tlByteAddrBits + 1 + val opCodeOff = tlUncachedOperandSizeBits + opSizeOff + val opMSB = tlAtomicOpcodeBits + opCodeOff def allocate(dummy: Int = 0) = subblock(0) - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) - def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) + def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) + def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + + // Other helper funcs def is(t: UInt) = a_type === t + + def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + Acquire.typesWithMultibeatData.contains(a_type) + + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + + def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = + Probe(co.getProbeType(this, meta), this.addr_block) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Grant( + uncached = this.uncached, + g_type = co.getGrantType(this, meta), + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data + ) + } } object Acquire { - val nUncachedAcquireTypes = 3 - //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite - def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + val nBuiltinAcquireTypes = 5 + //TODO: Use Enum + def uncachedRead = UInt(0) + def uncachedReadBlock = UInt(1) + def uncachedWrite = UInt(2) + def uncachedWriteBlock = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) + def typesWithMultibeatData = Vec(uncachedWriteBlock) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock + def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock - def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { + def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + + // Most generic constructor + def apply( + uncached: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire acq.uncached := uncached acq.a_type := a_type - acq.addr := addr acq.client_xact_id := client_xact_id + acq.addr_block := addr_block + acq.addr_beat := addr_beat acq.data := data acq.subblock := subblock acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { - apply(a_type, addr, client_xact_id, UInt(0)) + // For cached types + def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { + apply( + uncached = Bool(false), + a_type = a_type, + client_xact_id = client_xact_id, + addr_block = addr_block) } + // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a @@ -113,83 +173,189 @@ object Acquire { } } +// Asks for a single TileLink beat of data object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { - val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) - acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedRead, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + subblock = alloc) } - def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) - } - def apply(addr: UInt): Acquire = { - apply(addr, UInt(0)) +} + +// Asks for an entire cache block of data +object UncachedReadBlock { + def apply( + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedReadBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + subblock = alloc.toUInt) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(write_mask, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt = Acquire.fullWriteMask, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWrite, + addr_block = addr_block, + addr_beat = addr_beat, + client_xact_id = client_xact_id, + data = data, + subblock = Cat(write_mask, alloc)) } - def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), Bool(true), data) - } - def apply(addr: UInt, data: UInt): Acquire = { - apply(addr, UInt(0), data) +} + +// For full block of data +object UncachedWriteBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWriteBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(Acquire.fullWriteMask, alloc)) } } object UncachedAtomic { - def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, - subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedAtomic, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) } } class Probe extends ManagerToClientChannel - with HasPhysicalAddress { + with HasCacheBlockAddress { val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t + def makeRelease( + client_xact_id: UInt, + meta: ClientMetadata = co.clientMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(false), + r_type = co.getReleaseType(this, meta), + client_xact_id = client_xact_id, + addr_block = this.addr_block, + addr_beat = addr_beat, + data = data) + } } object Probe { - def apply(p_type: UInt, addr: UInt) = { + val co = new Probe().co + def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type - prb.addr := addr + prb.addr_block := addr_block prb } + + def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { + apply(co.getProbeType(M_FLUSH, meta), addr_block) + } } + class Release extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { val r_type = UInt(width = co.releaseTypeWidth) + val voluntary = Bool() + + // Helper funcs def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { + Grant( + g_type = Grant.voluntaryAck, + uncached = Bool(true), // Grant.voluntaryAck is built-in type + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id + ) + } } object Release { - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { + val co = new Release().co + def apply( + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { val rel = new Release rel.r_type := r_type - rel.addr := addr rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat rel.data := data + rel.voluntary := voluntary rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { - apply(r_type, addr, client_xact_id, UInt(0)) - } - def apply(r_type: UInt, addr: UInt): Release = { - apply(r_type, addr, UInt(0), UInt(0)) + + def makeVoluntaryWriteback( + meta: ClientMetadata, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, meta), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data) } } @@ -198,29 +364,62 @@ class Grant extends ManagerToClientChannel with HasClientTransactionId with HasManagerTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + + // Helper funcs def is(t: UInt) = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(uncached, + Grant.typesWithData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(uncached, + Grant.typesWithMultibeatData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + def makeFinish(dummy: Int = 0): Finish = { + val f = new Finish + f.manager_xact_id := this.manager_xact_id + f + } } object Grant { - val nUncachedGrantTypes = 3 - //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + val nBuiltinGrantTypes = 5 + //TODO Use Enum + def voluntaryAck = UInt(0) + def uncachedRead = UInt(1) + def uncachedReadBlock = UInt(2) + def uncachedWrite = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) + def typesWithMultibeatData= Vec(uncachedReadBlock) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { + def apply( + uncached: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) + + def getGrantTypeForUncached(a: Acquire): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedWriteBlock -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) } } @@ -260,10 +459,17 @@ object TileLinkIOWrapper { } } -// Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { +abstract trait HasArbiterTypes { + val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with + HasClientTransactionId with + HasTileLinkData +} +// Utility functions for constructing TileLinkIO arbiters +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule + with HasArbiterTypes { // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back @@ -271,10 +477,10 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId] + def hookupClientSource[M <: ClientSourcedWithIdAndData] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -336,35 +542,26 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { } // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = +abstract trait AppendsArbiterId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id(log2Up(arbN)-1,0).toUInt + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId { - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - in.client_xact_id - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait PassesId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - UInt(0) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait UsesNewId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Mix-in id generation traits to make concrete arbiter classes diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d2531a9e..c05aafee 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -67,7 +67,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } @@ -87,8 +87,8 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends // Queue to store impending Voluntary Release data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } @@ -117,9 +117,10 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -130,7 +131,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - co.messageHasData(io.outer.acquire.bits.payload) && + io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), @@ -156,32 +157,39 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val has_release_match = Bool(OUTPUT) } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cfin = io.inner.finish.bits + val macq = io.outer.acquire.bits + val mgnt = io.outer.grant.bits } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) val collect_inner_data = Reg(init=Bool(false)) + // TODO: assert that all releases have full blocks of data val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -193,21 +201,19 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact_addr, - UInt(trackerId), - xact_data(outer_data_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := crel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -216,72 +222,107 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) - state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_inner_data := crel.payload.hasMultibeatData() + state := Mux(crel.payload.hasData(), s_outer, + Mux(crel.payload.requiresAck(), s_ack, s_idle)) } } is(s_outer) { io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { state := s_ack } + when(outer_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } } } } class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) val collect_inner_data = Reg(init=Bool(false)) + //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) + val pending_outer_write = xact.hasData() val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && (state != s_idle) && !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - !co.isVoluntary(c_rel.payload) && + io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + !crel.payload.isVoluntary() && (state != s_idle) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + val outer_write_acq = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = crel.payload.addr_beat, + data = crel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -290,16 +331,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe() io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId), - UInt(0)) // Data bypassed in parent + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -307,7 +344,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := cacq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -315,17 +352,18 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) - when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_inner_data := co.messageHasData(c_acq.payload) + val needs_outer_write = cacq.payload.hasData() + val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) + when(io.inner.acquire.valid) { + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_inner_data := cacq.payload.hasMultibeatData() probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, @@ -341,9 +379,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back - io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready + io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready when(io.inner.release.valid) { - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { @@ -364,14 +402,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } - is(s_mem_read) { + is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) - } + when(io.outer.acquire.ready) { state := s_mem_resp } } - is(s_mem_write) { + is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq @@ -379,17 +415,22 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_make_grant) { + is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + is(s_mem_resp) { // Wait to forward grants from outer memory + when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { + when(cgnt_data_done) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index c28778aa..14575aa5 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,7 +34,7 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) @@ -65,12 +65,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(doSer(io.in.bits.payload)) { + when(io.in.bits.payload.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !doSer(io.in.bits.payload) + io.done := !io.in.bits.payload.hasData() } when(active) { io.out.bits := rbits From e6491d351f8d9141dbfcf39b30b569ede8e4a050 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 00:22:21 -0800 Subject: [PATCH 281/374] Offset AMOs within beat and return old value --- uncore/src/main/scala/cache.scala | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6651056f..609c3b33 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -166,12 +166,15 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + val refillCyclesPerBeat = tlDataBits/rowBits + val refillCycles = refillCyclesPerBeat*tlDataBeats require(refillCyclesPerBeat == 1) + val amoAluOperandBits = params(AmoAluOperandBits) + require(amoAluOperandBits <= tlDataBits) } abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters + abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { val (cnt, cnt_done) = @@ -750,7 +753,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result + Reg(io.inner.acquire.bits.payload.data.clone) + } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -803,13 +808,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { val old_data = incoming.data val new_data = buffer(incoming.addr_beat) - amoalu.io.lhs := old_data - amoalu.io.rhs := new_data - val wmask = FillInterleaved(8, xact.write_mask()) - buffer(incoming.addr_beat) := - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.uncachedAtomic), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - wmask & new_data | ~wmask & old_data, old_data)) + FillInterleaved(8, xact.write_mask()), + UInt(0, width = tlDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result } //TODO: Allow hit under miss for stores @@ -860,7 +871,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St manager_xact_id = UInt(trackerId), meta = xact_meta.coh, addr_beat = cgnt_data_cnt, - data = xact_data(cgnt_data_cnt)) + data = Mux(xact.is(Acquire.uncachedAtomic), + xact_data(tlDataBeats), + xact_data(cgnt_data_cnt))) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 6141b3efc53a2c27a50ce28ce3200a79bcc1eb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:02:06 -0800 Subject: [PATCH 282/374] uncached -> builtin_type --- uncore/src/main/scala/cache.scala | 14 ++--- uncore/src/main/scala/coherence.scala | 80 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 47 ++++++++-------- uncore/src/main/scala/uncore.scala | 6 +- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 609c3b33..794e3447 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -747,7 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) @@ -760,7 +760,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -796,7 +796,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_uncached || xact.allocate() + //val do_allocate = !xact_builtin_type || xact.allocate() val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() @@ -921,7 +921,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat @@ -947,9 +947,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.uncached || - xact.is(Acquire.uncachedReadBlock) || - xact.is(Acquire.uncachedWriteBlock) + val full_block = !xact.builtin_type || + xact.hasMultibeatData() || + cgnt.payload.hasMultibeatData() read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 6367b44d..201e67ab 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -174,7 +174,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) + ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata(Mux(incoming.p_type === probeInvalidate, @@ -189,7 +189,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -218,27 +218,27 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -271,7 +271,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -286,7 +286,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, clientExclusiveClean))) @@ -306,7 +306,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -338,10 +338,10 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -356,15 +356,15 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), Bool(true)))) + Mux(a.builtin_type, a.hasData(), Bool(true)))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -396,7 +396,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid(meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -413,7 +413,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -435,7 +435,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -467,13 +467,13 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -490,15 +490,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -530,7 +530,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -548,7 +548,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> Mux(outstanding.a_type === acquireExclusive, @@ -571,7 +571,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -603,13 +603,13 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -626,15 +626,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -666,7 +666,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -690,7 +690,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( @@ -725,7 +725,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -774,14 +774,14 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), MuxLookup(a.a_type, grantShared, Array( acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -799,15 +799,15 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e7291222..e58dd08d 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -325,7 +325,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), g_type = Grant.uncachedReadBlock, client_xact_id = tag_in, manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 32ccac87..3fc0e4e7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -76,11 +76,11 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val uncached = Bool() + val builtin_type = Bool() val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - // Utility funcs for accessing uncached/subblock union + // Utility funcs for accessing subblock union val opSizeOff = tlByteAddrBits + 1 val opCodeOff = tlUncachedOperandSizeBits + opSizeOff val opMSB = tlAtomicOpcodeBits + opCodeOff @@ -94,13 +94,13 @@ class Acquire extends ClientToManagerChannel // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = Probe(co.getProbeType(this, meta), this.addr_block) @@ -111,7 +111,7 @@ class Acquire extends ClientToManagerChannel addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { Grant( - uncached = this.uncached, + builtin_type = this.builtin_type, g_type = co.getGrantType(this, meta), client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id, @@ -140,7 +140,7 @@ object Acquire { // Most generic constructor def apply( - uncached: Bool, + builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, @@ -148,7 +148,7 @@ object Acquire { data: UInt = UInt(0), subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.uncached := uncached + acq.builtin_type := builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block @@ -160,7 +160,7 @@ object Acquire { // For cached types def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { apply( - uncached = Bool(false), + builtin_type = Bool(false), a_type = a_type, client_xact_id = client_xact_id, addr_block = addr_block) @@ -181,7 +181,7 @@ object UncachedRead { addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedRead, client_xact_id = client_xact_id, addr_block = addr_block, @@ -197,7 +197,7 @@ object UncachedReadBlock { addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedReadBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -214,7 +214,7 @@ object UncachedWrite { write_mask: UInt = Acquire.fullWriteMask, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWrite, addr_block = addr_block, addr_beat = addr_beat, @@ -233,7 +233,7 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWriteBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -253,7 +253,7 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedAtomic, client_xact_id = client_xact_id, addr_block = addr_block, @@ -308,6 +308,7 @@ class Release extends ClientToManagerChannel // Helper funcs def is(t: UInt) = r_type === t def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + //TODO: Assumes all releases write back full cache blocks: def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) @@ -317,7 +318,7 @@ class Release extends ClientToManagerChannel meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { Grant( g_type = Grant.voluntaryAck, - uncached = Bool(true), // Grant.voluntaryAck is built-in type + builtin_type = Bool(true), // Grant.voluntaryAck is built-in type client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id ) @@ -363,19 +364,19 @@ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val uncached = Bool() + val builtin_type = Bool() val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) // Helper funcs def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(uncached, + def hasData(dummy: Int = 0): Bool = Mux(builtin_type, Grant.typesWithData.contains(g_type), co.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(uncached, + Bool(tlDataBeats > 1) && Mux(builtin_type, Grant.typesWithMultibeatData.contains(g_type), co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { val f = new Finish @@ -396,14 +397,14 @@ object Grant { def typesWithMultibeatData= Vec(uncachedReadBlock) def apply( - uncached: Bool, + builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.uncached := uncached + gnt.builtin_type := builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -452,9 +453,9 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](uncached: UncachedTileLinkIO) = { + def apply[T <: Data](utl: UncachedTileLinkIO) = { val conv = Module(new TileLinkIOWrapper) - conv.io.in <> uncached + conv.io.in <> utl conv.io.out } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c05aafee..cc4755f0 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -260,7 +260,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) @@ -268,7 +268,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -355,7 +355,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = cacq.payload.hasData() val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) when(io.inner.acquire.valid) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat From 57340be72b7bb9dcdd8f38b759df4044a77812da Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:11:13 -0800 Subject: [PATCH 283/374] doc update --- uncore/doc/TileLink0.3.1Specification.pdf | Bin 0 -> 172795 bytes uncore/doc/TileLink3.0Specification.pdf | Bin 123869 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink0.3.1Specification.pdf delete mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink0.3.1Specification.pdf b/uncore/doc/TileLink0.3.1Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..23666814d9e8738924e59e5f9f8872e7fa58d725 GIT binary patch literal 172795 zcmb5U1yEeU*6$lExJ!Z$?gI=uxVyWC4DLEO2@)*0J0Up1-6dFpdxAr7cZUFPIP%?d z>c02t-YTklX6?1Q|J`ftu08A@otl&k8z+z(jc#r4U>Xg?0pI{QncAWW39)N=J6izQ zB_U9Vy^|FhyBfsG!VSO)yRLyIB7$b&X#U~{dNKT0z>D9Dx2uIC^k44m8WwI&?yhFA zc-XjVu1;o}7Em{UAvFKrLQkWjO#K z(7z?H%Q(TNfLUc=R<3_o*=0BZAnt#C1OY%i|N6)U0P+4i3?`mk*}~igBH`o(Fnn1s zFDD4V2L{0=6L)lUdQtX2%m0S~ja}2-6#9?tP_l8fLt~e4ay7Sb{bw)-EKI|~3<@yh z<x??;0ATIfH| zyTnkuesDcybHpK|)O7z5%eo2M)pW0_Oh~DZ{ct)`-4vrh_f1PG-2{2xVS zI87fSa(VyU4nHsmvzPndp>UBtTQ{D5M`W#j`b2blca!sUYIv(~c5u`6m%wR%@1&os z^W*b_>NAPN@$>c7!|E549~|#oezXK8QH!nn6>4o@X&}jARizKjiGNH4JQ8lm2@c=@ zEZS{j1ykaXW#`3g@cp-momXzuPMdQq;D^JOmshXC^ zNJR2?xw!pn$$S2nRoxM#AH7)EXnp@gGl|Wm{4SCQsW>308z|oE$e#;_&n#$|(#oo?!egNpp?m=%Hh_wnVV>c9f`B#%A;<`!S{Z2s6} zJ$d2upP!yZX)o?N^fR_sPsrClUhbDWZ z;Tn@N=+WePf0?&XHyS0a%cO?2jD)ajru47_CaU-p3NjM=kcbLD;0yQ#{bj{`18v3( zW3g|Z8WCh0Y7`$;Wu0oIGxh&6N%ba_B@3|7nEWQb{Y37|kd}2WD73d#>;1&V%;riM zk|%z>WBp}xNV&4&f(Tn-fNsjfU}`T zT_2T*Ki0t8f{6E~ywaSDQ+1e928HtISB)ofr{G!k1&!BHRi7aYTVc2MZHYYWZI1zr z7m_zL-aa1?)cuMcYK|ER66G`Kw#cQtttM^|q$)tUCl$c^9@5%R1@+VO+=uTS-Av?GuuspJTM$X*@H*LhNJV@GFK0A6v0IdZ^W96H(j`4B z1p+A@+SIMT@H@>Tswg^tjhjiX?lTzEf6Md;5K6Z|whyzu9LAbE-Y!oi=Xh))0D&Wr z!&{w9X-5OUaR^TlEVhP`w~mkT zH{I5qE%)c4o3dp6+{%Gdzq$}@eV94N6~9h%BQ?+J8#F%au>Tx_v)KY2Hnz=ATg*8W z6RJ>gxjoqRghJY_g$(7!fRdINlKm!=G~moo zq8q%{y1gW3>0;!Rf~nvyy(1F|2TR^D#3%@4@I0;p_qWanzU28FS^&Z0b~3K+o^@W)~PJjh4b=3TsN$GwPS6r<8wcJA(L}^w1B0JoZbmj zkH_f09Xk&JC4%k!SAJIkT3b#dSAGqFC7;k$1FoCdE~(~zT>J`^Y8A<6<4P>V|7aYM z&TG<3`2DW)Ol1)J?DS}D3!A+=GHjlI>0d}|#;4l?Esc_u)B&+=wUJCZ+ ziQl2`U2eAMO6kIHLPGNkiz9ME^Jx=PC!Whg2qY{SxM%O1PsTS%m+g>4gmr*QohB!u z{TNT078Hg#h|PKC^3B{_o`qLv+~NTejbSONbsAuu>i{X1j7A(gmc;#M&CEYrqp zxs&RA?4ot;GhN0tQWY1@5kj6OhS}%FxyHIp3v)779w1E}1T|wkD}-^3OU{(*-5~>3 zYwBHYpLkD<^WjS9w#D&KaTwmV_2d3PxSt)~&Rm>*I?e0b+zT=Pw;tW1>!6Zbe;+3*NYZG`u}Yf%Z*#A*NKw823y z=`QS|e&hC3WmTbS#Y4tt^owVw%`BwdGIDxel^U6Ks|0MvK(NtycQVES_q(7jme;uk zf6ZOL4_WGnFUkTJR$pZr&=IE_(r^M5v|(xy!2Xd)^p(fSL;`y@Ov_lpXy4*vs8r{& z=Bs%ulx(NigI~+Ji1Hf7h7jjI~?TXPsoQR_TompGXY z1;=t1E@>Tze}a>;h6~W@MvhA)u#O1#K6(VpiBFcPu^zAy{46Uplaz3Avo(*ga=p${ zr^m~0Q=xCX+{==3^&a@qe@I1D$NBSZ4D+VK0Sb;ho3!<(fvamJ73D*)FS>g86k>Q1 zj(&)Gk8`<%Obzzr;b~zUv)cFUtPlmUbd{@_1ZYlFq1Cnmt*xDKRH5^UqQYij;a-Um z)iZBXMwaj!Yen?_{E+&>qRvDXhquNWQp2d!U|C%``pg4JuAn&z&Y7u!4{c&*#80DE zssU#&U3Dt=y}fo{I<`b20dWFP2DX_obmch3?i;(%?LCR;qvRo|l-6F5T%i;;sxb~B zk9A{10qHi+^r<6tYnF+0z(B~r(+vgr%5Q~(%?fuJ;#UV zriT>emd|}eP=}Zq)I1{KS=g)oZl`MW7t^Xw*++Zk!%1cvB71VeVSe9id>Cf1N0z$S z@0g;dhTURxh#cut9)7^&mA@r9`nouajFsSHV3;ArW$_vy7@}I z06;x336?s&mK;aEDKtTMqSXG@7yx2I-{cr!fh#RBu3LNi#kbWyM&e>PiZ{QBJ^xslQB6yX6){L* zq2!e%qo53wMT{|R1OUjdwY)Pok;XofFyeC6wNV<=LNEBBLUXhFu+rUe!4YjeCsPmx zbjioLAek36eqSu!!d`w#M5JoEMx0y`>1ka{Pm!2a3jaljOB};r%aW(RdtI@Ttc%dk zUg4_{76911U{pBar&ilPmiSDz=*5pdR5OZ_q~gE+230~34=ZG%M8WY-+74Wt3a@@l zl5tqj`kXFmu_P5iBJ}*1;-UrZhH)Pv7m?~zLt=sKrM>KtZnFhiE0*c75WDo?v;2yS zh@+77P&>_Aw{f;5+^E4Nq7#%Kmu!Y)h^`bednXJFb+2$U9f@ngJdH@#e;RZwqH)3* zf0>3TmaS}9^Wrl zee9c*{LSjRkqAj&XAmJK63w|a*P}`T=0qt{8mVuLRapf>ZOS^#5!44mYg-i?L$s91mhMtI$h%o8^1bUv zTnbT@Sa)sTh5`NdK}O>@Md3I(ae^SS==IwI4A_0imS}94_*k-~%z&p|jGn{o6_Gi| zpgcMm+$Ulaz9KQelT>dP)UK~~`n%_HCTAMAqpiI#(;{X4n`l2@DuZ>T_sW;A%INEf z-dQ>}XIw@qaMUKZYp!{3=DQ^nz?u+acQ82IkM83G-p!G3Y&~UAGoLEhS&wltn;Y4ZP3F_ZJGme0hcWIA`KA4?C6JD!Zn8Eg z7&qxoosN>vp`=19=23cjQ=*6sJlG|oT0A!Xj-oCj${J^|n;lo(FV{MuinDd08IH#f zJ`a$~5uJ*$^i)v~8BVwIxHdXd8~u!}IYE}2w8yAbXjssv)8QVMDWIAMz1_>(-FO-?9ZZI!%v620j-O?2}rzj3gP$I5Z`L;r*flT(aW=u`yx2cwuI46f| zD$ro!GmqPOpZh2vB2K?H@ub=4HkyfTkVOca1EK$*VmiRqX(ZhxZ-it#d4e+bMcUU!B9dn1Lg>e1>+ z_6suFuzC_QjBVGf@&LmSqb}P_!=|N|usMztQAbI9UuEdOw0!Oh0A;;mob{>}@F@nq7kIr;bsIHj0uPy`r zvKpIBy#YSf(si6ucjJpmBl-Xfdf*CFB(8t~83Wz)9$+hyeyv>E6L>LBZHnT_WWk`$ zEP!opXi1gq&fIKh$;qop`B;`{GsTPxYWTQj1yVXeuN$5lGr(>4c^*>H6*AM8Xeo7P;k-)>?zCdJ0Pv!@+v^ok%S7{X)Peuk` z=OLuAG|Q!^HBTZ8CSyt!O8?kTvaS7W5h?m-BGoJxyU=*8FV9zeh@`hBP|kqk1Fji= z7P7&oZN&x=BAu;iVHDb>%RyV5rg0<+KsXX^&FE$?tzaR^I?x%OVHtPl`d%fzei?%uXPn)yfRvNRU9v&`ADuIayeh!Q=pRZCGN4xc z_2ynwHH5O^V2I@|Dz&kD z%s}aW#DQEYT=U_Q??1(8!bX>a$Vf=Qhjy>s4Vj?h(4+_>X?N~YK!}HgoGFu*GJ&$U zoIq)!3QoG7xV%pGim;ZN4{n~PgR4A|U;#Z_)(O9Ol3}EkwOddA_MSD6L5Kr? z>~~9s+V`1KS@h-eZ!v3Rx?^5ng$Cio&qTJf7M8Y3D}J}rXzWcC6A3@=lvdzN6-!q( z;y*~uiqH~C1L;jd4w?8oT`$cf(M+c-pd@dS3B~cO2xO(CcDH6l0P7!uKVC31gqId6 zG$A-p?50#c95mF8Ec+~vfOs+DHo+XD ztf9U3RmtfbWh*&ZLHZi^n*@hnK6$EiYXuYh*f*N9R_}PL`}}D%H7UqIE?E-u*r@vI z)rVg~nG51UKlU{;o2b4-istqDUoUW~4_g9@eCeP#f_i^KLiSN`KM>Vj9+r^ZXy*VH zmOGhG`B-%h8kE^?w2LL(+^3;>ckpc{e~#N5wPQYbeB5g?q{Zn>URQVZeCH&mB@@`( z!~R%Q>&)^)NL$79+YMJ|bm6k<4sXlR#~*wRpqmHxWykyG$lOExb6{w#vc;FCO#4K^ zGoVQO0fbO88~xks26Cj6;IRyE=G3_PdKY&*HO&ul|N z0eSJ-z!&g=sDsjtU+*w}+KbVW;G`Z1iZJz&T2WY!Gn4Fl27O?|6 z4`&t0!`>AGx9^EQcThS0k(mx0Z6T>D`h+Rv(~N?Jn4*gHNwrEP9- zTaj&Uk{>Hzf3{iJKQpChP@9eX#;kriD)2V9#ZNr`BmYQb#Mv&$mLa?cbn?@B+M_-| z<@E3y0iS)Ap4R?4Ir&^R>e;aSAL0VS%x`&fwIHvyw+zuX^TLgw&9uNG&h z8O8GZL${`@3up4`aIg(+&5EV@#4fKY(lH~?Zx=J-+5!hAsExDRly$Zp)Vs4iIPP+G z{fh9VM6chp#KoXZ+2XEP$3J48eD8i29>#mMSA$i9L=SR6AN-E#u#z^gIe^lQ^oC@< z{!fc28FL{^#*ntPxhFl~gbVvgar_F4ObN9kGFeIa4-smNY8DU;!(BkUpzf(sr`8be zE32DN7^9?qXKxTDxgHdsZzyj)M*8kz_hOrS@RRiY#nD*7+-BZ*Ti-TY@Oe|bu-%|p zwua)DS2+AAID1WbvU$<_yYM%1e#jtNqQ~E>Re20;vg^i^=C(jpa{=-CVSnYm+EI8} zB%@1nKC_uJ_64dYWBeL2gB3z$71I8U@GZ5)s?T~@1I_fN^9!Rqs_>Efd2+T?pOM_) zRBW}E0uqYj$hTycID7;yO+ttmxm@{&^K<$4%=pNid8Xm=%&=LxI%vj!DRD{1>eVmJ z^x{u5h1ANpf$QJDp0k;=@IWAD#5yFyQQAJo@;iDvqFpP?>f^c%ogY`Xz96W~3pwbc z7G|tBE#GHtYodCme0}|eQ`nZS_fRAfs%tf0|8?v22SF^NzBCJgp{eE^dP`Y-gARP- zqHmkL=+Fs4z0;)$8MoWj=AG@nynEK3{a#OxK4Ct06S+oW?xfCNhs$1=4&c4h?agV~ z9Y(0%T)heY)%c|!w9W2MOcF0nx*+O@D|2X_AN!Bi*VCSLNgPG(B776DdQ7{wH9WCo zBW-fX0{HxPeMdDPDm`k@8T8GONR}G8x2~KQhsXXyWyxswRAp}6a`~BLdp}JP+3|Hv z=k$8DBj8U&e=n$)#Q324_*i~bEV$uM50l?e{|c^ZA9emlT%_u%&WyKv-9Um}8O~q# zx*%8O@BO-&%2(UD`e~0dqdh;gDqnkNx|1nq!=3>e7gim9ZGAky2=}vPkWKBVP`m9@ za2?l|5WzCO9*b$Ypy81fOY(cww0j0MZ#}|mtJ!`rXa&2{$W08en`pG$5IA>I&jvz&#dQ6L zSPK{_5i^{!?wgS;iLOfY_RBlhGX4cidvUiiucM#D>e1x1H0O@Sab?>hxk#u>_>l?( zUt5`rj~U6a+hKv*!)-Oa+sCk`JIK#a)FCY6v$ih$aJNR~;CJ4GII%S%pGEkNjjZT4 zJVkO>Dq{h6iy<<&_hhay)03$YE(0C3sxqEtVKlG!zoUp9E~0c$!k$GCzvCy4hnPdV zQJdz-;G8*vy$7y~4BNM<7Bd}p*(4Xn^cjr@MISUay!H?$D-JY)1?y=n1+`Tfo^pjz zP1)i$;+k7OkUSQ}P!0rlmdmqzXFc8PpaiSBYvbAObrX1>N^ae(mdnALOqvYgPnN1C zCf$3M#=EvVB4W_0J|h#w6H7u$k!fn8DHA~Cft%o1QI#dC)p&`Mu z!PH)!U$e9goZ`3>q5|Pb%wmLrJtEKUy-)KuE4c{a) zZNrxme5B1wG!m9TqXd;M$5VG&t9QYz`7NU6DOAP1CXI2uK>X$eDEgjj^#|{UG>DEA zb#|5<&|12`Pf**R6^~%HGi=GWexZZ?A;AS9BD)7d2sOyBz)qdpbTy#)vaQ8qJFYEa zb)ea8B@g>`a2Twl?|B_nF4K*O|4{BaMBU08rAc&E51HEdDaRipqO>ti{=1~Q&OP_C z3}t+t3uzZD>zNB@5JOYNFkX~gT%)hs%kzEsT}h~&b&9eh=IjKk;a*()pHkjM32h}( z%t2#D6UjPz;&{ZpP`P6XC)uSHXh*0e>>Nrvb6(DgXf&9~p9e`~A~2xa_ZuZz5gR{5&7;^ox!%K+A{1PH*SOq=b6^|kLL`ANDSTq0T@)-(b7;(cC{cWom%76K zcOyl3qdPx4s4X`FRD=ECBD9;k=OJu;flg(oqrunZdIl78Ra-(xY`tqq-x*zc0xcT{ z%#4nI5^~yhiaTjlpngYHArr4kX?ytL-S<{?^K#%PWT}?rny-5NM}X3P1ftif9oXUP zh&Ks+m{L3uN%hRTWClZC$)Nk!c{>uL2d)0q11j+pr^j{Ed7^${^19CNDO|OCFWZIY z!-%eGqEx=v&mny!bMiw)U@F0s`haSdHwpmzt+^<_jjx}~+a54A(maTXy3O2`%NN`< zWrcq*yp}j>)Wi-asqdREBpZW_;kf{9ZI8x{#!8WVBpDJ)ga|+N*@dSFfh&_fh-sN& z{&>!7v_eqtPi<11#=mIK-ftCp9gwDM z)mx77kuMvwPU-LRKMh!Hp40OPraUm){*9vmsipKu+n#+I(!eb%uRoo2TJqR!z0bqz z&8Ms#j)R~xhqH*22YY}m@AcIO%FIVpZ@bH+`FemhSOtf0YjyRe<}N`)KmU**#ZbIX zOU)4=Npkh5`*F{|{aHcmPVQMEDUR#W?vyY77xu{s4a?E&z@B5Oj)y{m`-#3m4sEUW z#Um$`EJ{Q9_^bJa%s4pNtD`vgKcoY{ly3s0esvSg;tzP7Zb_oepq-G$FvcJBq={xO zMPZX}VAa)_Us9QduD+&5ChD`B3mp}p@DAHsCuIY9%TnR{U;OxO@{w+EdJ!60@Dma| zBf*F(Y@^Q3s~umak&n6TKV8>uL><9hQGsDZ&0JlQi|551K276Ue}!siMOB|em1t+k zYpHsOht=l<2IQVoBs9vI4!h)Zxdwcpi*mT(= z%joVj=y2h)w6Ny;k$2JkxH{q?=OaMu;j2MCWW^2<;)6pJ8_)LAIa!E>*(Xm|#V(mA zaQ#_YcjW}*QqC0a&~YiP zC~Hb4K|!C?q=lU>)9v-o(*AMv!yyDj?WcghP~H!stNfXrIZj#Rxr9UwU$EG$LkRUm zJ|xtW=}3*5`mC zJ`L)cV#I2))|zl>-e!1iNo8%3?rKn2SGYn&e?8u4q%+!*q}7Rz_GIY&L^sPXVtwT2 z4j~I}4=f2o0LclhT4ZQ5P9z279oi7_LTJr|#i1-#dQKD5tQ~Wh0Kd1mKOvYMVltmQQcqc3W_A zQqx`RiSgN9nx#9z0(Z;VjE-Rv(aniiXir`d2=9PHZ+hD?^lKL!RU)8zmzcwyr+Wd^ zGwacU>VXj7Yst8>z6TzY*!-46Ce*p9gpP_s7#d4BBB;ms;}~A5e=fu*Z=IZG#7TT3 zyDoP{$8$ea>KA6}uWVaGP7I69CabWP0(8uyFV|miMP=cr!$1i@6xLRBHwD((+XSH$ zT^Vk>3nVxvaV7f`lsbJGp;+kE37QsLDbJ(t?wIJ)oUnOO3MY2JYD;mtL^P(ZI_$N# zA#GLFjG&&(X4Q-n4oIDo{*I@5Nrj4J58u$S7>BLuOO_3NjwVLgnq|hI6&iVJEYBzi zK`Xrm*3Y;OL6Ym{1$($y1=Yr4R&;|qXuR+l!as&)m*I`V*_dhPIRMUE_g#d)F0=+X zC7vRbve|S}p9bus1mqc;^EA27Bvn7)aULKze^%OjbtJsu7B)DIczJ*EbC^#-%U8iT zKf-}{J-snMY4$h661I`c7OU2MX>)BX@i8)mo~2vPNmwuAZV|4j2nnP8@x^0+E3={Q zDWV_}(baN%%>swL`}zZ%!`sGi1(Y((-ghXHzTPo&GL`pAkarGm$0Q2SmqKeG@l6;S z-{7Tw)lEw5&Nx@cZSW(8`{@)~bFJACG&*jG*RR_VjKwf69EK^!om2|PeGoWmZ08xg zy?Db$cAa>(OC;)nzRx5Y+1f8|E#Vue{_@&EV}fklM9j1LkC_{jU>|So5&kN$se;aS zo|;%7x}SL9A%-RB~tO-N99ASu%jiVKe@-m0fMK(}xHaRqQ zDGN6xB^eZxvks8wK~mnf+g>_>cWylW?*(|38=W5B%-FHihe-?Aw1pc`vNn|CJmWR|^YG z3s)P<|8pJxpzQuz$3JPi|IqQ0!2cH=FZq4RwyTXZ)X5dV^-nbK|EpU5hrN@OmwJhO zsVxw!ishxWoMh#tlp)UltB(F**7SCRS~$o%S~|U8cN!K}Hn5w+8^9%QQ9hCCcE>F0m}eVjlLFPO*pMXezZKN!f* z&GoW89xy);2L18CAVMx6h@X?o7{JX9t6e^RAQy<^B{MJ|4qhM^H{S~}_m8hIDv+0x z?xWPO?&X=J7vG8#7 z0J%9iKrlsMAn%J04@@MO2UeXt+^`AxdHDb!epnO_?~4o`4sO_xFrgqYHxLZs1OxcF zIDi~{{Qu$3!^I05i4$h|_q4E;^Ya6F`C#tcumSk^d0_K_d4b&guvPwh8e`-CxFi3O z&HpXw|0~^S|G+k3$e;ymq<>O{|FEb)9ALQVf7}-^M|+5s+skPr{&F%v0bnkEAefh% z|KDc?6u<@ofqX_~Zle zzYO~S!$5L!@Nja%w&;IBkVn4Wep=GMcfYmXTz7QPb`&H+QRE$qN8#aWaTCM>LTPDf z>4U4?1LA@U_?J$J;50f!?p2h z?_+Uko21y|%E_qjs_4m$`_7VE>Hbowrd63{#Vfd1!sQ5XpoN8HhhCX$X*ekgc$^%A z-_KI(>QG>D657t2tgaTf2An543e? z@BK){?yWY4RIW(8&s74b%3fI;ZcK&pD*yiJ@Jb-dWC2adBUoqL82KOy={Z0TX>R9R zXa^_8$=z(mucy+d=^>RnBKv)QQN)#ar^_t_Mw9!pzhz0;2sGW;?zI$tIsq7405~N4 zf?n1`L6%2+cWEiHY|gPdlnOF`>61_mPV@vE-f-FX?U#)9)O2bQX6(pR2tO-j{u1;& zqBy;OKtg2bp$c|EQN%UHaTH4<4kif({TN@7_F>5kgV-0_hDs*Rq36wgdaM#p_IiD$ z6>LAp+#r#x72szE$vNdUkGOg7|MprGNr#EZ1^>KCYAMKzK?vX%e2ZHxM?7%D8f>!? z$Wen2L6_j-Uh@gtQ}-8L2RA=C^|3z3ni74(k=^h$o}l@QPfrDWV+X-|M4qC)o)yPt z3t_%z)01MGx`32zLv|sLrr@0u$O(!hq>O`t9+l=jNsKYgUrdZC?DqsrZ-Y;8R|2$* z_=snHxdPTiI-OqI!~Rt`@um5%Qc(^Z(Cn!2`xZ+GYW>VJSq}?BtuYqk?0cLq=8e?f z7>9>Z4nmM2MC%`elkKr=`08D&dm;xs-6vLb?K+X_xKYvKDah+BUNwrLnJi+qSc|4M zBjWWpK}7cNSc=?&2d9h}XN|sh%=CC!`fvMo-zZCzcSRg**&IvG(g@L$%eLjlhC=%< zY5cK6YBRe`A~zZanh@3?_Wlvhaia`>U+pQKEM0G$F0;yguVKJhA*2iWf z4LT8G)&1_(iFMJ1^M&M8RN(iCIDa6Ns|oi)I4VD`L*l!A_omGIvonky!;-cmIN?NB z`!1rs*w2uUl*TCOBTkc^GF+4vAA>ezIkBQjex4aUc_f)J?I#}{yyK6pCUL#Hm-fg;MrLxARiDcNGZVr?EWT#@<`SSyFaQd^E(VwBeXqPem&Z>(4rBmuV*||^7M2mM%m4R7i z(u|UNh{{v^Ne#$XiP#g;t!Aqh4ZqGN|?u)p1HMedG!GmL!Vur#@w6owg6< zc|U~lWSVgtHEr-ZwA}=Sy;tr|TP}_zeuwq^FQNDGC+E(tRB>D<%S-qfsM z&TLGuhVV62wI>dP+^Qt~?DG4^!|p5p@80uS#p*?*>c7b#GZb|jwTIz#b?34=|PPg|%+5F7b@i z$eUA4P&6@E(l+=szK8Z~m(lh!zU-6>3Y=sU_yBMTR2 zS)h5I@@i)VI$``UbLF7mVvp?|<`i_gOAiA_ZUOEY@7>hi9FDyA&dDK7dys5T-I!<# zY(IX+wTlyKMLzG_ILtcD!9R3n5G+e#Un7dfBAIw+>`GV}+X70%enVX@<)!CGdHj{~ zdo)JeN=S2bHg5SPs*bXb4mL!c2T>g1O(bjJh0%StxOftGZQidvOvsu{;yfNCIx3>E zALBDQ$(x+73;5)^v~W_==8ve%=@CE zX*JMu)%W=mcI?N?vl>;A$HR*ah6x=zmtx&d_y<2~oY`7pmc;*F0zO24WHw^4CvYUB zAXp*5T<3P_)MHqo@Ynpis+~yk?MK;X+uDp=%QCZH@a{nd1@DSQ%8%7@>J_h1e3OJH zzt~e+n8eB9^L+jxrD$rt9@lr5<6Bb zB4rP(k9AS1$4{Hrv0abm^9(@V*h^~9PVIK!INGb?pYLbhUFRYyvfyF}6?YGjbyWbdOss(xr>|JU2q#`2O~4sXpt9tWc1nyIVU|HKjS%c| z-%tb}{4r?nH^LT`pcImgNr(;q@>cFGDg!+3;Lpvscfo!Nw0TrArqq_yiQ@x1oBUw$ zCy7n~5uSKCZ9*2UKN98bAd_oG7L`dt)xhe&Y(cD_WG9`~py#IVx#u~;9dgFD)+@2b>dRFu9{bHEDR%xf&)=KtWNo`EmjcQ*tEI z`QM^{e5aIqJxHrhORCasBY+swgUbIp=e?sK7G?L2IefZYmN{O!5*mboiMrn$jftwz zoSBJcYg0i(oOzI2L%Jx@7W8R^wr;?_IcJ0ls3EzX;B7AQjSgLo{2K$NIh75qa-T9$ zyyX=9q8y99Qe}|u0A6hpE6uPxslNDvJn|l`u{qryec6ERX-^&WEjgC&d-iq-h91XT z;dV5>@07X7_cpJWFjv3l{K38DYDZ+pXUjrvFBJSa!rGLo%*K|kVWh?Ho#3_}e1^2~ zV|!;SW>B7z$5qPKb&4>i4>9wXJt%a~-9Y-cT%H(e6r+dIH(lYuD+r_TVX3r_AMy&PTnwoHk``ND+bPQNNOCe!3KwF!m<%$w(RMG%ORHz z(lI2zrho3CsYDL8)VoHxBFH!O)E z3tZUom%k0$D3z6~G?|<0asK3+pI^fGwJ9AKTZe*bCFxw%-v*kB@5-g%n;e?qSl*KN ztoJUD8H9U%sFsdtVAXuf(mgBmTLOEA@Ac|=!J9XnEi>!USJH5B^F}dyl^^1cL_6*3 zNgnKZqe4mO!x&0&q7#hTCx6usB&ty^20yjPx%V#!kCAAZME7o0LUdfygnDB(Ts=Ce z{n2kch&{v8kBn3_wBAG$+NL#BOJ)aQ|J;NpFoWu&3t+eA`A*}JId^4a(|yYTKCTG(F&!JN+LtoaBXnj9DQOZ@%0D)V|>HMz50 zj=7RNVBTY9_SHEZqwy5?Tr}Uu@nyV&bXU9RpIWODKWgvDH8XhyESu3xTQqer5zE4e z#-K<@L+ApGl;1Hx-l?DL(>jOcc}LO?7OZ+)?Ru3`-(Xkt+z;}!RetB36(QsTofIN; zlo!?#CE+WfzY`jsQ#%x4_%0yAe2%r^cu4pCAxVug@W(rnU9K~czfpbKP%aa{V17zY zSrzZ*)VJ;WsMyyDa>aa9kh^ZB2T*V!iXu*>gkID{#6-|UR5eO>^hEGPcr{KgTrNc} z76eX$t;a9e8^sOR7RLz14absZL?Z1o2SP3hL_mVDhdWpy7%i9ucL;X~r-(XwLj6uu<(Avm9*zz!PvXlqt@D#o~ynhjS+p> zN=&lmV9GeUI56RSA`u$uP5M^h>Ni)?dW9L|d|N;+trMH1uGbFN=INLjKEws#G2KFj-?H_TVr39~|GQDA)th2o zD7Ai+Dzogk%BN_*zqF~eV08K<=V=5Aal3l_HLit@KdWM>SoV9S%EpkUUOS<2l-A7c zal3I8-k$n)XKAMKb5Bu}@|d!A=6+2sX9IZA`UAta6M%y>qK0OTT4NfaRi(>%rg~qq zFhsB+cd`Oy4F!Vo8c{e1OFZ%`U;u>#5mj6Y2Zs?}+_@6TkD`g`TD3gl@so4^)5kPZ z|M9PLtG`&TMGpub1P%J18uR8>69^wNN%}uZX&gF*M?AWV*H9KkJo4ntVF#Le2ESn- zi8yLC)n-zeAov;C?^5r@NPw+wuaIP_%^D$(s*4e!qD~M^&8!7i*sgo6o9dfX>Ro9Z zY+rh&HZw=tr#YVP@#pNdbNlErrWrEc7sz z!AO~;yUq>Noz$S%J=t>yM`mH0S0by@eLlv@I|Nf57?lo^ZN5;--HZ#f0m0%|ZDde}&u%9_6s>vOTh?oH!gi@Hq9@ z>iKL3XQd+MPqYJHu;ZtPniHI*2+tDV>St5zKgLHcD~>&+GnfN4gx(UD!g^uY`^CiO z7%O%Z^pxKTY zIi&Lr#LQQFkB=W{J1U9PEoFk|9aU;Gj}8!bO=P=~`H?O+z~8s=;rf=RaI=(U33i~E zqtnUzMy*84%-K?|4N{DgW8HmI6@-eYOu50$7^D~kB{~y0*@&9v;#Li|Lm^DmI^4be zHPp%pZsyE$<`Vt9^2b*>?nN1+8Ls-j4$kI_KBs8VGAm+iDbQ&&)k`k0WT(^vm^z*| z#^iorXiecdFcl^+nx)gKr}4Yw%_ronq~%x1ep#|Aj6)X zQ{rUw;=n0o{%K~Ky+*u!8P1TF9#iqRj%0QPK`YhKhXCI}@pCG$GPjrtDT?~{*Q8?a zr*Ga)5c@nj3Ff!(R=j5D4RwZ;Y76oh*%7Mcd?*r`j_3YcX0|X;TE=6AOwj8tsYF>& zc(s;ac`c>}NhkIbQ}LkHsQZ@}7_V&E0a* z{cPvV=^&o#0a|VP+Q4tf*?Y)Vw3_kdED7@5@>KAoQ%Q;Mp1rMzNU3mnH6v}3<TC@=ZOG7iM$8`-TNy-PAqG$ zx0auJ*!%NJj~G>LXC#R(ff+Y z!n72G#~ zX;C^e;)vzWp*;EBO8vE+;RNawcWX;}bnzhN+%#^VM-eMD!`KwPp9Vv$Zfp`NUM{!g z&f;;GLT z@?S~|bTmFk_gx>DPwgGZ73M#zzV}Gqs+%x-G})EI;}iq9hG4#J8Jsxyo9v1duZ);5 zWBRMY=c|WQIIn?0aH4jZ3XM zeja%{ig1Oc?RW!4s0enu7zaq#um6%lBrTchPgzTg`fjwDs|-VN0WHH$6h&e-Asd64 zpDlHTy87g#x_Z3MWIVr{?Mlp`ac0qSm#f`Py0Cv0Y3luHlC*K}u%NDk@zKMql|S<# zc}iPl&0*{L5)=ECkRNnm+Qm#@XKOSkF4g3G-#9i)!dQP|%rlR2kYgo1A&80z?@on! zMLt4<5pF0unI2@O*`meTVJ#*UlIC`u4azRXG#L3}7F(c7aUP9Vm0IAoLuYg2cv-;w zJli8*gJVkdnfBxp?K9ul*^)PP9u1?7PMfxZW^|zpnr}36G$n~DsWi84f%;HG^4#`Z zhQo3r#UgK}y<+KvwtYBl{K#WMw2ZQMLEtEUx@px&B8r?)qFlSG0tE{W6>9b~-OeS4 zIcDEFvk<{IHcRUU71S!7C|x2zTecx+bVFoF_{5n!)YKB1m5i@td3dx8O)s^a;^);vWh^_hb>7Gc?v56zsw)=V zOx_l$+D*qG;&U8G*5_57h*j(n3U$DRF07XTVP~}GEdn@$0#Wg z!K(y-GrrhAFE%{LEfpi_emVQC8}02fmU=RFcqo_Y+n*v_<*46jd9i|to&8p#@BTqn z$?v4zY1v`VJ#ld7w@I7l(g=0?&*uhyvJ(&b@J|%Q-v1AA_ZVc!vMvsPx_jC-roEce zwryj&r)}G|ZQHhO+n%;<>+N&S-uvDg_aFD`8&OfKR%TXZR%J#-tg7FWD>747%^>7M zfQxFzs_*^oW$Mfj%fO!@)kM(>k%AJHe#2Pa82%kS0oaBoeZ^P)f;`?$+}(x<6PJUz zk@9zNIb2l5?_h5As$pkaDVS2STp$d98qv|Q$Gq=fwJD?im|zi+M?x$`_l?m_pxHl-GFJDE@Y z0Hwr}FTt-L%*Nz%v=Mg@7PGr;b%AYhP-k5Rf`eMF08NVISkJ$Og`TjZoP;zg7V;pWd% z7CjHd3XV+ns&BTN_8%BQCLRZzkmx??b+YXgP#dSjKXr#KB8%&?o{w6Q&ma7Oi0r#j zy)#+^5g$hZhgufy*)c>}q`ZTYI*XK#kC0k5LqMd$-l zi;~1?>x}HBQ5_(^+$e@PxD8#bRg|e!VC3H=Vr~5bQvt}2bKJ!uiC#I?ZxhYQ(L&ez3$pI+6#PqwT$`Pd%>Y3Ob*?N5#qFAxX z-%5;#t_HPo9~E!SZ|UyJAIgi$;zs=KCq^(o1EtrDH>;G);zZ!QpykYnn)LpVh_z=_)w0R@cL@;H;{~%8DFa z+yHLsM3g5cBy_rs1{$H@K@Lbuz3*>4%jkRg0QH32%NvPx9sK-sW z%WV;(c`?w%`-|mfaO!ddW*+OkGG|Bb=L6`Ve(7OvVAe})Ijr4KlMoeD0Kd4ZC02pu ztWa@wYM?4(LWzlsNr6daePX?$$&cARMn@zn-QL21@Cs~pAnXL4JPNAaJjfLUJEif- zbuV&u%7gx6Br6JC#_vWLISHjAuGyv@f<jh?_0>a2k^0_!#oOb z;M!zwyS9NamO~CAl?BGOFTIg%Pg_$vV;smd%hKC4s;vIZf&dwB(}NzGQ+ z+tE2UTqS;J|+drALuQ(MB%ERyzXyl8?jIi+>T27EgxP9*|2m7dY5nAiL(1iY9G ztq0AZUKnzrCh(;Adqyt9sqQ}63KA_JzDg=LSMLS335bvp4^Pso^1uRmyJdD7c*k3A zn@C+Zd~3UcQ@OL^{KrMW^U~0T7KhCWpy;4XX3EPR-lN`YF&WzEsH$H*lWgN>v*#(o{ zeeGf}!UU`Qss~aZtcl79LmkoE6S1_|$4%MlEf0B_Bpw?(gO7kX-Iu{vcZOFrfdqfJ z{kh$d`Z>%RB|;*a)MSc0rj}GBsXF2L>NYl-;N=vaAatR?QQT!P4s9B2|G8{PjccKA zmgHMTJc3kc!Dvi8pfaEmCygBmYB1gcJ zD5X=c6S~uL^Hm=OXmd8t3qNgRH=)a_lNI>UvY%sRU3%`A6-iMg_cbXp@u=~*xN^xW z)CxPKD*f7Ag1s&!7&V_THCFcC3+vQND*I#VP3)a#9(55(_;7fFzF^?DJeolHf)3pR zJtR7#j;ji;J#2BCGqQ4z%w4yE_c~}Y8(^p!Rt-eShwMRucZ)=}-Rm>9P3Gn8%43F0 zE^$)5*7~q>e$s}!?XJ=+^a4W}+hyp`{a_0IC zkxYnxM`R9wOw!tPzPV{j)TXJN@bD+%y$f|N%XtQ=Ib|A}EDq;m?@R&798v);3a+Rm zQho?5=$jra5oshu(0D(kBJ+^L&`h&&aAm)(Nt#ZOPSrN!YXRN0B#c-H@!3#43P4N| z(gFS$iyxHD@T*hAsi-Bw3=;OEg=#TqK6raPwc~-xj6Q9O7Q&TOa#R~uF?1@JOI`EE zFQ1O0OsaHUo=vI7j&ib;1b9@jA>ND;DM@@jjbkzr7}yQs5ng%-Wn-_kD6Xng6`Z~N z4zGV4qONMS7L9Qj3c1NyECrga!t=Z)+c}L@p=+to>ZnRRyl5r49I!K_*^%sR)|ih8 zP$o(0sm7{fl$2%6B|xZsmx>@!c1>#}7o0(w)@B;JP=d1|(FK|0bIb=Xq$8&1Pz&0z|{$%>16;cLc7vx2VXg-l zXL8w+Vn%0%f`UkuyvM8?PUx)1^|GFDOL2=5o--ih8{j$P+s{d5&wAMvBnz<`2yMh? z4TU|Y8HgNq^|HRO29o4kKM_kZqjKZR)7`G|>QXB7DMpdVKnV+yU2b)2tMQNbld)Sa zFNC>Qqxa@~|4gpW$tu!DT+hf(j3=uyle?X+(DZLPpDuXN4(6c7?aTry(P-85X}I|V ze4I$LQhcg@D~%DK&|*NFOx;-0;2)s-ROyRXA8xEvw;wp^zgaty=2X*7s7SWHNp%ZD z`7s7q)6u)w2eR@VnYYFY=d+EcZBDG%a-;1AP!||yX!kscX#s3iP-28^aRLm3D#St-A<6hC7 zZsX^S1U92&{h=xywRfi&{u^wh942*>ws;dEzRTKW?c`zcAW@pB!6rF1C5uP6!q~!G zYoo?pu1K}Hy~D!AO~B~%#vpHAOd_#3y-vWw$*FlwsWnY_!rY1G%+%b*sT7r=bbUfY z0&A6~lZ%#hZGFS%S8+;SZ7mo(g&{97WssvX6zrRqx>iI*Ky|9W&kr=cR|)^HU50Gr zYkK=euro6xPl_`nO$1HBWl~O3PRM}awLrUi0SZRF$fNJ5@;Qis1e5;mZ;^bZz#>0l zjM(h)DReIey>2;Pt52)YWRFpIejv z_zk_eJvB0}I@WSVKwao#KD-53UtStrGGaI0;($_aOS!GwdB2c&Bi$PKsntOtO0&~; z*6;Z0eK!te$&EX$KzdX5hUZ9Il5{(xH5-NXM^AJKhc5!uv72=@TGii*=?2$uzkrXT zZUF?%c&83lv%^PX8dPc53G&ct`?Sd}Gcfy5>mm-|n`#TrMV>!`${LsUDlts5{s&s& zqdl~0d_$`LIq{ZNx2*i2ew1nx>&)y4)Jt?95a#Pbq(4&7)6qib`@`9v#dhV#(?%jb zlezZLc&Y{`wJO*HUhuTm4Ivu1kFv?0s)3y&%sBdv>E5N&RkO`41jaa*Zgf^Xz7dP0 zAeDOk*3`KJViw(=9=hJcVY~4*v7@#N<<^h{*vk)JyGA%?ecsD+zITT!V-vVYCK%;* zz3KK}eq~L!#+Q5C$q-7Kxiea+3D5i#S;t8gYiN5?{V6K;4!TpyqC-|JpOL+6G>-#v z>bDXxqzIs6Hi3?oUp}u0o;?oBnK59M^a*pnb8p;M%l79r8N%*>WQ{r{;?OPwUdxZ> zD5mQ=b0pjP+m5=X4|IyKm4W{UU;h{L{>za5U$F9vvi=WNsUbIJ+DD7feoFC>jO)88 z20b|43i~t-?(hN>SuYLRK&k({)m4@6pkB<2#L&i|dM@m6`-jYg;Dr`oHN#JnK+76Y zw+(^bE8u%{YW+ly-{RTPEwADFv7Qp0(Q?Kry*rx62u~&hvEYvjkpZy)WROp>b2WO>)M7P| z4_?a?%M753!HfZoaq+BsT=MxjtYt~=;|`@Shv0%&sn`0O*Xo7lb;kUCua2%VLB}&E zsMAe;n^~93wCd&+U2rt%{(oQXe+&2jC3gSgWdFE*TDbs%U=_^GB7Yu(|+;huSr<{#j*dL_fO(ip)VZ$H9_oO>GWR{#4>%U zXXY<0{jY7dFYx_O=9jPk65kBeZ2vOP|Ei0bnVFi6?Q8dcHdy|{kn#Wi{3nB%j`pij zY^?Y!f7SGNiN9<94+#IOUe>?c{CBCp_xk5}e<%E1;Gc~Ds_3t#{FB4R$nb^Lzx4Uv zDPP3=>m0ry_+L$8X2xg!!sCCRJj-9~{hx$?HvZn>pM-xl{$ki)*XCbf{_k&ppNefzL+wMZf=jWHv^8Hs*i#;=enCj)e`M zj`ho^|9#!)Sn0ny`#)FU3*ys$NqpunwoXgS!1h0p`u{UB;lBp^e+K_wNc}%CC;ttp z|1<9Y7f`2VqNZnI_!8^?I@G_&I@On^r)HyL{0g=Ck5Fg*8i-%8|Gx`$7CI(s)~{HB z{|l;rMFvnaG10Q(Q!&tgG5jy7PESk!SBL=}EyGuk04*Ei|Dfvs?H2ksRcB=W()0f} zRj2=oL!kd3_tdiv7x!Nai!7ZF*3t(xVsVWoag0(>RIP=voy)bi!^!4RS7 zS>>to4D7w3S-(XN)%2j6-V?OGVMT#O^Y;~Q9I9|% z_mzCJsOD*TuD%u^ja=Q(gYQyKxKMV_3YB7||H4?E-{ApYJH+(V#fr znl;se+H-Bop(d<&gFS(ru1B;*>=lXo7)`##N4wAlR|W-rSEm2|ES&6bu%y?{HM^cm zop>3=>PN;4!%H+c>xv&p_!7#RxK4bc`vX~s11bZ?Z{9K)A2xVi$7`&>NtGlFdeYW~tGxTJh|UY6kSnbI`>q)sC&npLQ~*<{P&7Un|KQ!= z@}%#w``k*}bf`?EyN@0D0P`j9_yBikjH9r9AH9tz2mV2flQy>t=?(S)AV6jLlQSmr z2qmvckHe0(3*odt#~m#Y*#?_l9=SKnSdfLEyP*d+UxoG4XT}Av;WMK<32WDTJv4Y% z6&zW!cHhmt$@^sNxyLC#;h@)sS!cy;5>~pgK zxt0~;jJDREJ=|w~6!gv_klHszLC=U@gYep_Fh$z)BYdjg;oQ$-0V<;B#RE~pTH`0; zWAL^1r;6{P?)r$gHFqLckO3I9cB=r;q{eW5G2lc`?IRhR`w=|guzH+u?^GopTL)dQ zIEc`QPJ3f*SsIvMm|0fP_fHOdnNp_y= z1IHdwPu3p6pez%rqmK^SLs|N@&pp3PHqr<(1$DQq$d`tH5hXKQy5G3R!Oo)c2tpog zD{SPY+*Nfd42&I1vzvKrzip7uV*?!u^)t+;=FL;KaK$6Pg+M4_^-Jx zzEJ8}k33SJn?PLD7QAUK1UE9r%0#}mF6u}ACmgfb+a*QGov3ml!?H%Ps3$i0stb7I z^cNPvkem<}`S^R+9?jg2^|!^)$s1ZPzz?-TS7bbB&*2LE#9Ux@ijElKNqJIzokupw z#Q3Ws?APpUt$YWTi3P5zG*v9WbZ+(N_l1AveRv@BwO~{ddh)TkW(vLzBR{CD;7tny z4utoaY%BP<;d{eG(dWkwrEJ+RNS^06DU(=!8CdbXYE~7oMv9ys(wya0+w=-Uagd%# zdOW8cJKUEYxyI}11i|-ZCu!RNRUGhbA$J8CX6`t_pinAAiH`GXP^TcZ{KBpz8-xuaL00C7>_o!UY`N` zxvPOZ$G1ls>4r@sl3C>&z#8?P8U7w|4$%)(CcfdDb*Km5S%^92jmnu&m^Kw}*-)sJRR!U{$Sb9@3X-FbW+-lk9#59r1?P*~enk z@hI;Jd9%l~ga<|tN42cfzrogPo{gitk6E+d^?g9OS~Is+i^h*buInJrA~lK`9OW4g zr9|h4HTT?1B|w?@#u$XPKzwhT)!-t-d46yb`K51BfOj~!9p=3HiZqcAb;_U61)H+B zUI=Mb?92(Ma6!Sm;B0ODp zP?#N1WiReOV?o{eji3gXdeb*C&5O!1s^PY=f%jg$j(Yf7J^0Mu9L%T#y`s!f_T#x5 zMZlo0)^bYZFWJfTDu$%=3fH5poVHSToQ0kOuXmJx465)9@?d7s%gj(OA{#vY@-|8w zCimwN@9Q`wT=rHLkl__s1i2tRo$ls@%@}d%pQAJPT|;dJo%!+jl(-R*<75U&ifC7T&JtsZd63@wVn2<^jg$R!^Udou4>cFO$6Sj|E#+y#VeM_(BvW-m z*KqC}G{4}FqeXv4x_Esqv3fmkY9r`ZD~=}f#?++k*wXxAKxBj+N-dsOA)7|v@of!9 zFi3a-QzptjqV}~6q63C~rke2tRPLHl^eCPE8mZFCdU0rR@vwI1`vB5KxIo3@wafR2 zX{;)Oy}_wNXBl?CEN*A%51?Z6`jx_CzXQL{O8Ah=yI`0E5w}y-Jop^=rZ#1?JlTIm8UAvCBh6EiBad5FuP6+ zbTYM2;G|rH>|;MV7!wI^Gu05c`WBU$CHX*|L_)Y=%*(pq&aFgUH zuHMpR{>DU)?)!Fhyz!tb(kHEFhBueah^|SV-A(bf{HA;_aV7nZbfUI>W?MX$bWY96 zXuO!iQHnkEJ*B;go7Wq<=f`+9@-8%uIAn z8aHNFVykqPjkFV3=QNjBm&+UK+e(Ro65CSpg;Y}+DM^PSX6XsZ4lW1d69~Nf6c0_F zrxLA8mRAeUqRoAoTbAp?7wEUgCyMvF_uF^qcbTW97nYZ%7pK?9C-|q$TfA$pgHLu@ z1PCzp@1r1nzyOe4YEkrb^i}o3B(?czEE%b?RNqubrgJm)g}YMn3YTU>zKPmY(erhO z00*7`GeAR3a5D(p*Hi=*1Zg8Wwy=cDIuw6a>@j!+?%B@)O)n5%FW}9VP zYOgxyh;p@#2Irxx5e;V7uY%{{t1As&C0{HW3(XGkMebAvaW9G<71$; zZjSZ=3;6tGrL+#-i*A2>nc*>O{arDxE^MltqwekQW^O7&^PBYxvD{<$uKj>X{diGY z7F^~eTZUulZAtDg(4rfe2KEHI(&v)FB6%6&jM@xBQZLr$^$}>&sjN5odG_arkq**N zMYiR$f53fC1lDG91mnW zqgIB~>Qh&Sf!@M&2G!(6&w*NDhtBjKy8=-9cJ;b1!xwEKaYCPN8$Cf*cH31WZUi*x zApe2w(Y6rjU(|H0D?&aBnzz$H-L3#TY)iZQO;O<}hLgfw~)vJsF zva8K0e%VnCM#2j!=`Ti$S=jACHaE8gEGcr_iHGYSBsZz$*FXz`>m!~F`RRYDgRUa| zAx@^_j|^#=omh@fG4k^UWu7m2SWsvlfj}nCXt++FX*IV&P@rB=aq?zg#xQ9=kk>{#n}MpPAJs*5jN8+JxQqDz9N z8&G!z8UkRxVP6qABg&*CvWo7C>Cne>iTnB&fF1t{1VTunn06t!Xrdj$5u$*%jCRPUQE=CfGBB%s9r44 z;T9(w*w8kNI~eH}QF@@vmYr>gItkB?<~{OriaLSPNYx(84M&SGy%=l>u>+B0>J~Rx zysS9pzCtd&q8!tLER~74gMtj~yr|T?B-On5ZgB)-o+Fk_t*G5ewsa~lvI&yIjD9Nr z`n&+TJdG4Fl_>Ll40fvUJ0tI$31BtZa9WZ*i~DB^F?2`2VXcZeT-p$>LD(|V!*{j8 z-?c{8hwd>Qz0B28<_v4Sj@24xun#1!92~=Nd6gf2IC@z%Q!D6aoUe%*`bo=CcOoyy z?Jv>;B9CpH2v|$dpTgdVKj^iFw6;&5!8&8CdNm9f;IFCN zDK5s>p5Z#9&^3Hez0^VUG7@lyuVR-dVgkgO>Ayo-uS&Cctv$h!Qa9^ zA$rDt`gslW?BLz%y|I3NcPg2Bg7l2d+R3_|dxPN_>)O(~sd{4b%=DV*+D3nBddqs_ z_~7aq_S#B%GWY;|hJ21!U1<#~?O}I~dhK^zZ$2|=f~aXzR1mgQgE;o8IkT^Vt!;mM z1KsdPDZy8TdMY ztdFG-7c?w%15&jW_aw=kl2FTcCe56jaJ}LVl{I*HRpL(Fl{K|{`<&MvO0xz1Z2v)| z)sKEl+zj9;JR6a@E}AKFLE}n8btm>pt~xM#QvFksWkjVy@xXv=B#4>)!AFCg|(R{tpV`y@z)al&QW9o<3 zTHEEsyA{sU>)K_z+tsw&CEmx@u-wy75T1wo*%N7&r{m6E$&^f1%Ra}(4UW}wWRv@l zTIVj+=5^ww^O#fnKF5bqr(U(ktF}(t8q{jzh*icCJqu}@&~3S)RGPjaqh&nqRnfD2 z2d=or)NX;$7q}OJaZ2TKdke728 zbYz2aqWf5Epl#{_;@<3`tL8b`%v;&~A)V4fIp4KN>3k7x(P43Le_m+F&y$awQdz7L z39`f8uHyh}1x1tpA_vAvu;jNLiWWZ}WDp()LvwCqTv2MMUWIW=GCB!*4TQ&0N75CL z5W-88JZt(jeBI4EhTKa71g3os-!M9tWQ(25Zy$2)~IamR7bD;rGQohAAmkUjOj zQ4tc-qY+aVt-M!qqO@}x2o>jEAi&YmxSU-&^}dGWo9X|a0zs=CL-oe|7B)_IfGRi*f6O3L{6+s!HBJ-FDW`# z(?0q~I6v%#U?hrLcY(%uZOj9`tBcOSm{RB=%?k(-#da3oG+7G|&cHB`p58yWn0+U8 z2V;%=I#DW3vZ9cCQ?L#kvKBrxrb$3zO`qVkL$jtEy425_MVLRKLn7A({!d0ds;^2) z0f8#D?Zd+1@@rW7=u3Lw$v$`d-uYSje%mD(jA zab|Am4Te?QeH#JVgw=X9I&{Dedtx;a7W52J;a;vRoFy40A@K{s<1UX1;>q^HPbTq# zSbok7frFd1ye0XHrB}S)rIG4r4JbDv4a>9w7W7u(O&tWDeOGsiFyP}L^=e&wii)d0 zZe-_kaU3HM|sE`aM?`0)Z7C% zEgiTzzaRq4ALj}FJAN2GO1^U8VR5z?dLTNG8gv_#5lH%>#UVT21yM9I{k3Jc+JvKN zV|(iqq;=+MMsc^h$XRKPyn$80u!i9X5|&ehf@7)N`ID(i6a};4jU{eejmJC)DCBGL zBOqEtU`yq`GX#EFk|BBD@m4CT=nhw!UGIx;fTP4#z$?azX|;aS(kw*B_utlYct-=q za2?GP#v_{RA8;?k=E7$}v@gV-UD;aQFs^7!-Tq7b{igkz{fPa<{*qHWk1(Dxb92fk zcEkKAgn{_K0bg%|2^8+^+Eb6ob7V#p{h<-#GkeQYGW$(3`pZHxyQd*tfrB1q7Te#> zG{o^4DA255^?*d=DYYY}Dhu%}-#cpBQau|c;*K@vI@#Va_#r|wJi;TqU~mQLa-Mpz+F6C-+{cq zu|}5!9k)SwWOY_dKOSwOx+0iB%?jL|SCCs8SM|Afvm53c=Bp&r%&;^SsI134ptYi} z><*ftrd0!`!3; zK~u(62EvG;+=-O8gQWb%8rV-v>qhXkzUr2Q36a>!EWtoZ>b|P^*E*ZvYt38biP0;; zmAy`C^RQR6-sA#Sq-Sbe_T*hKzm^ZQl%8+@|TIqr$!A;ZvPyQLZ z1e@gAMS=XF|I zh|#{~y8a$glp0rYp_g2%Q}n90ly~8$vS2E?f?P}@>gX1q+ooQzo(4xQWTZP-T5%!r zedXA5d0~KSvOqsP*zk*>>+!|4$!JHvI+|C-omx6s(>J!vX&kkVX&_K7$j`4FTSf{D zhl4R#S4U`w7j>c(^nAE1=-XF^n6{J(E=VkhCsdf`-jx0+)mMxiEo>~EFwIk~S}a{K zRhd{&fHhTf*fXd7_2*mTh#_A;2gvtkB<5c>sE?rm8URx$Ctk8hhrIO{NBnjE*Fe6e zHm;m9503@Guph;Hr@Z`&meUnlL9(agmJ6k#S+bgjP0^)Kld)9(l+*+B2pL*Z){kW* zh%t<$V$G#(-UwsqLSnwmsddHmDCyU~pfds%9TVRsGd}j@aoY?(u5C6Dq~WM87n_Lc zxmy_HseohlW-W6_{b|~ur>2s^*w1_O7-MH+HOg_GONygJjn3H<5*^D(K;mt;2zYea z;*;?bY)??Gf^E0tiUh|hLKAQGJSdY$AQI<{yQ|`F%^1bmqy<~3;^soxEyN^#jk0pm zs*p3RY`1id6YwNnzb4RMCs>P;+mUt3$3r&!3T;+bCK@O}Aqd%$4{a9BmDlga6ROn} z!nGi~DV{|Jc*WZYjbThKCefu2(5HMJ8wm!DOV(Cc99Q@3V9Q=mVEK^2Qzf&nVwSeA z?XJ+5Gdl7@#BXw$*Jlfe_=|f0*aHHe*wz!&lOrLBy21t`qQsgbf3c$Xq&@B=_uZZ`JJ|a? z@pBP!wmfBg=JvMC3GZCz4>&rslq;caP?sd1FXWagqTqL8NL_|OUJDi+#DWT~2aYM6 z>4*9_eUS;7iR6d=d3?GiN*nM8JWfJ|tU!$?v@lz1ylf-lB6b;2~t}m$$?uD)&?Y`WRpBCg?77=93 zc{He*4W1fwP|X>y^LIx9P8;FQ08H8-5C4V(Zy@;`E|k9S%7U#Th27e$eMWq^_Cans zgeBqef@mVRQSIModZ`~Q^Wp5HA^NV|Sa1C9O!iP12e8JpiPH(9%p7?U+)Damr@D!h z2S!;#!b8>j5S?|fB7FqlStTE4B+Wg(y9=c65iBCAxo{(eVt30^llje5LpjDJ=0o3$ z`vJHJg9vb6lOroC{VS@GyB4m%zUE>-w%rRDHW2kDnMOUV!{Nq%Rc^L4Y?h_PYMIBl zp)&%0gWd4S>lfL`M<(isrvfbiC&crERps@m^ex@adoC{VcUhJ;j$tyY=nTX zP&h%XkU%Eq1xVt$qIdTwR%76DWM*hq;@(g{a!ko4U#KzQ!?Z!(#LM%Uj-@V$@Fjws|T#dFbE7Q%kvciNR8qY_k;}2s$#i@3uK0gvFm(V3W|T%P?C((jJo8?v z^qS|JtW7BoKKCAA3?khM*$0U~B%n>d&n7Fe$4QaFBUv*^Wz?{gSc}cH#*BK9nQb~Y zbxmkA;1Xq-MIQ|9Qb5I98642^M89gj#01$yCMA(h{g>|(*Si(XPGbTTRqcB0ND`Nh-*FcyD0N> zgwXWqd5C=cP^Yg7ZKNj@x?I#@pVo*qHx!xxoC`#!Pm(MJmc-*IW^D#`zMeB`1YphQ zMyY?3A&#+^=*KvTfq8;^h$q;uEr_LFlSV`lFaOR_*Q+skJbEmqEuCa&)R+|OE`?qm zqCD>ypshornO3g;+7|R^iUTOoN!VzxZ`1*{#2EIx7FVlW7xLA zbqF)Dc?8xks2vJL@+V^o>q9+-UuXAODrs!jWYVu>-a4T&ax(7N6|QBnX;6gGgd=n8 zS@@ybkS>Hj@31Ftz|XcF8{utxxsDS>jhGP+-N0N$G$Wfs(1b-Ud^D@UH5J5Y6%sH{ zw=zG0I)|>qJK(f-elqgID8cQ00!!ybLs|Ah#Ww&gZO`M$>U9=^h)c4)fHwiSzfU`X z(Q?9Sj|-=NfD{^jGyF4AzVl_yF2ufZB!+wfb*qFl14knOi*f>OvHKq`94~F}l5tAu zl0^|Rp>_x9iMv5`M>&8jeVeVx7-Ss^M-sF^k?Q>>l?7t#h=PBL&1)w)Ht1-3@ojhk zxSn5$hz4T1m4lZ%vf*-HOboJ;6ojecSSQ$^(w!RMorI3!yWQ1cEfCz|L^ zG3iv5Rhpi2FUS_r07B7SRu4-;o9IBR8i+y%#x~_L*Rqgp*T39;?XUX?>g_OTxkF6e zZp>z07wk4yESWekjjegzMCf)QY*~iDk#AJg zTh;bfb$@^O=iBOpf4a&8UlgC4;YOBc8cm#jpxY%u7ve5F>4|A%;%_Ri_lRnavz8Js z55VfMB$FxwP?>NzZ(AxN9WZjap;Jn@C4kAY{=?IIRv6Wz7J;y{4Sc1z9c}Mw zVDXSd6z@d(aPL2=3x((~K9IB^i(9khGZh*>dCl^U`{~cRKA0<1l?P!5tzrGT6IL^v zMFx@UBxSQ|4Wnx=&Cc46o)nHx)&su0QT{njx?SgDOgIulyHG5UB$D~anCE=1+DY8m zA+v&%(cPrfwuTRkSriOjuqs^CjmB`OtRsni|3YW_Si>j*o)VL;PHNMTnD=g+{RqOyj%dIv2`pY%9H_${BYGiv@mnb znbyY1(xCcDKEk85*{IsS-2qn)Q7p45A7&bo`>**2K%Bs=0U-S_NK0Ua~X}M(eOLwuZzCu&ya3L=}-G zyj3h1aCDY>;S3ZSCju1b^2a@jd=Y6?<|C7`LzXmD!B(3D`) z_{{>>vH-Y&ZIXD!$E4RbqxDb+KF)_)|4;Q=;L}UDQZ3_K5UIF z!<6X>+8W;V6;#>;DAQaX$^K&`W@qP9g0xe z8|Gke7_FgVK&F+?uPwW1O|GaU=Jx`0w}h)b6^*32NMRjNbuEXAqRsLF%5#f0b_gu zr6(%oLW(IY27~LZ@z7XAeoVndCm^7>CiF}XsgYxgwrak~FL)YSTWr+NXYSp|W9SoT zis6^9rs70eB9#w9rV--E1_p>h!i{~&&M5h)#rU+tx`G40*|i()TZ1}js)Sge`eja4 zbo$To&y}k>xxV{9-4T*S3!>?#u)+=H^2c=LiT6Jljc^k-0wLq+;Fs2_{Yivk=gH~g z%8vW9!RBAA{C1`a9FcJla-mhfkW7xQN9Z%6XJB2y@VhC$Z>#P*_W~xDS}Cb3B1@Wr zh=P4!sl1S)Omd_bl19e&?;mzMd*WErY^6@*sj?l+1j?e{nxC*ryY00V$9JZJfH&*;1;v@N?^mtfuc(=8zG|>5D zKZWf+_$0VDWgq8Pr-q{_IEq?r^VgTyUV>(AtSFl_pcTh}n!9bN#3*zQ>nZ{ExW011SXZlrs(bHqGfGOER*^Xqy>H;+| z;6L7i--SrU{^ArgmQ*oZ>j{OFeG4z+LP7S{0b&FKC98Q>Npw%AmFwqA_Qeik;o#PU zC~n3oiE^&HRfXJ#f0`yTJZ#9Dxw6Gle%#oL$vdFLRDM3_y+Yn=OsV*KJ@8C36(|#X z?F7mWXF6xK(tAZ>;&fbz=+qG(*I8scpLl~-9!2b_A61ErwNx&5tQK|p=kNoKe^ErZ zGp2CoA`xbx=SIigoZi&EQ3vCs+hZhwmcxa-JeIi{9ky-Rv5zA#&zk!AP(N%Y)m%*4 z5YrYQ5jeJcp^KsY!x%4`RuwDhE!)Az0aeXJeP2~XGy9ZsDBMQdPq1zxeR!a;r9qAx zkweNg&%F*3LMBvVrY;|)k`7$biMLw-Ld>267?(+oG#};%rQ-vI0T(qP@|f%V5Bad( z+VSz)TB!Lvv3>$OEmJJpq142<37k@I+N62QI;h=|rpqp>x5ekMZJEqfN%jjKq15{I zw7Sm_rA{o)kJFYZFENua2_~ypHy%!z7b~_`%{ith9`w$;<LTXEk?q&bYYbK8bSScGd-Invf;c_UI649YZirLnI+l?4!nQVLmy* zw~e#ka+p0D{jZeI9(~y&;+JWEZheN}rXZ~do})UJbBl+xE{|Kt-#Y8pGXK0^T*ZLU z1oc(HOl-`T3e7apc)XhlherDz%A)skXtOsn7$+5=G^iE;4-N=6q@v&yE9y?D8=6&? z;+kwd%h)r;R$RzmfKip5y0U0@+RH#Ha#JTWRKcSBjufzw6gZa-19 zy|C64vlCYa>B3pP;`TTRY4;#=+n6bGq!*=bDFV$B+gcY#a!Kgq;hHYF!Cm??a)3s*d%K|KCsz8D5KL7cjb z5dmaD2wfpK6=y70-36xKuTrH&C)cNm%E2&EF&j`yrnIsEJb)?$IYu)k&ls+sI~Ppa zj2#AFMRQiP^l(`fSFr9tGsCyUK4;;;RejyTTA#xD6HMj;;YN`7%u?-i%@#E^?ni-C znutl>pxnu=h>5LuJUcQf);N8#9oMY9Nu7VKx7#Ta=dHiD5g$V6HjFyh$VpFs!ScG@ zn{$aS*7WPTY6yp!ipfB}Z)@#~1CnI}!aVla#=CA)Z%ru9PmrOXeA2QqWH-{l4OgC} z$76hXp|#1t!%LVMRb#oX=oX`9TAf**!4vQ8kkFwlW|GU9DUy(r-%RD#jAQ>l#_l;f zvZh@a{$yfJY}@Ww6JugKnb@{%Ol;fg*tYG7ZTrjpoO9NDo^{Up-uJ&=dso%2+Sl&g zYgJWWzckr}sfDg`6KpS4FI5za@%LczL1br&Z1WWB3=_H2l7dNgAoD}s24?Hx34ZVo z(Q}yAI6752)oKHVazh$ctf&bx^LWy-!Fq8Ai70`F&X`gJ=sc2K^sekY7R*RdmJ(Lr zx`tWd;)Ww0rg%hU9Hi_doxeth=qdDx$x)~^JDx9o%nt`s)>Z;+9y1Rql!<;VX~_12 z*wI4CFo^}n4Yr6speoDsq9?pte3iE!q(@Qj z+j{LFFXHeeBttWx+*MB)8-niR7J+Vpkv?ns^WpQ$oQJ!ao1O+J)%J@ksmV6ulZyV_ zY5L}8Jn`z`_%KQdRD%j8QrZ*v&5 z4`z^Js?=1g(o4$cCAe6zr9@ip)L4u;^NCYSO_c@`ArkAt^YPsE+tm{Ugxe?Eweij} zrfU2$GjA7q&P*A(braKaa6?_l^%%LaQA<0InA8u)RI#i=Igq;QSbhYI%;+&goD)B- zp?gHC_Igs;@X|yIA(0DahV37_U~at2SYfWP#83TPR9aNp`QU-0dpw*Savc@-$LhHe zMyXA;Y9tj?Kt43Kp5t_sctabK*L)(IyiG(S75U+9>JBia6;&Q=@yxDP~9rrdO5$E{X6yt^X5r|K`JHqnX z$B@PKwYIRU&3t6Dt=lIiPA{=4>L?pHB{Yw+?bQ4MVcXlSH-?mC$F@4LKH*&aEQ7Oj ztSATGvmJR6yXea`6EA+Hn>UA8j|9EMU4Uk!I&gK$ZZhCH$Rj5P!zD9++Mw6mQ=Jn< zaA`NSTgV@X5PTR9`N*TW*fDT+l|O5p@6LI#Q%$bgq6Z2ifu-K;mTOE+`wTw>qh-YPwag?EdgQYqTfp5IbncZ zzf2Uwga_aE8Y-4j9tnhsj%=X6go{q3Py()hLjZ*rD-jFOhx4;xmJHA-kLxb1tvJ9Q zWpx(qS+kddKusZcr+f_;PM)J5N8qBvQ=rv^1{>Qmufn8W3)M3MTX3s;AU*2ykb3y} zA>%rAfJntfxp%nPTD=st<*R?I{`4!((fsZ&w-}FN5+}oF5BFni4+S4dpGy~(_jP~t zRjP=(k!<{1rSPlpQ#>R351&7qY#^#8bgM`_*g2?$aTfyd$|j_XDpdj(u_xEo)|Jto zEfZ-e5qCF@^LJVip%lf3X$JGC`6-{rje9=<1Y_w2N_&tL2HiI8eNG|8)z&o@?G|yy zt=HajxaXd8y71L0WzratTPfqB9h(MBE41kG0|iTJ@g?j`Iy9Pe#sNC^=78(y{y@|a z(Du@BVeU-wc==3ZMn1Y^w^{Nc<)T`kl8IEEm(|ky_Q$pYX@&z#m*(k7-2~QQ zmhPo3pBU?k-x+oVR)M$P8=%3EQV8kA1=0^wM#`;X@XAFZ2b^a0Z!tJY=}BvfZ)Tza zGB-P=6cUwjocYYx#fAFVT!#|U$vIIimy3UbYZBCUA@gxK@C$ZZq_kAwCKXFr-+(;u$>zf zfP&lkLgE(!n8rbPo5}TcZGHB}LFnNRBi6t?aELBr%$tiol*LCn{B<@@*-NH;{O9A5 zL!WG#b2ck?G9X(Q>vKiHjb97vR6ELt8`*Qe=VhKB9=QuO_Pzt$#JGode;|8EjGIr9 zUC&Oc%fsjh)>6rNbg5#rROh8;hL8hrv@OvC(BTgeRz&FW^VttUnl-b10;NOwChd#} z5P=lDjYoR2qhIBII9eito@Y_Tt@_ZQ-ome=_DPvn$WcArY|C5?Lm?3?onG}$TcAgG z;nrK-C?Fz-!i>op?>VnUkR|tPQ`okhBsc=4%nBPX2?z2u(GAEu&5#C+QFFFG~-Luvh!L zi-Mr!O9K3==cjv*^tdS*hlu$jyqbiyzAKvcO!1W;ChcMuH~u3SmUY z!g3ORp!EjV0W({#ekE!$QvW3erOx+@8l5|VR=STDA$!}#7xP7i+iOlow@aQ1 zoOUmcn`+nZDBqM6gEGls12VCNx@VWMZ4ZQKeswk^*3&rFIyN}lC3XM&73P#P*Dl!6 zD~u%usii_x4dm+3F+t(p}t znAd*&RBej`nwI`-{srOHIS(ZiFL;!RY3_*sBPWe+8MboPABgXCRJ}w-KF{fa9(8o* z6tG|QxD>R4{s?~Rr(+g#+VO}3t+C@Kdg}-3O2r{Z(E8zTVbI%98%hQ=&s>Ntp8>x$ z&>tj#MdwDsWHL(UqXgHavEY!P)zfkW*{u-Yv7e5@Xu^edV|)9{Z8D3jX%V)YP4IBj zlIH!c^0c|scUf4D+ju3Q6xQ2C|4JW~S@jFf=kwP8{7l(Ve+=DS)Unj9Q-`H>$9-tV zuV>Jd6h*u}nyry7+Cut9&BDn-+Cm7&;FpC?#v=A%t-C|zj($aS#zycwVtyK?a>Nk! zrM_>3f|OQKtQi;f-zeoMN322L1~i*|>wFZt&8mx`vH1X}fXMfX{9!0Q6nA^S>pAsZ`3^J^#LJVvChW_sw>@I2ympQw}x z+fZsG1SnHFJi3Ke#*af$c&?(Dxsy=KB0Fp=tqe$jDFPr88xEhh{`ejaEt5%d7Y?EW ze_q{LUySPJD8Rj}np{Zd)->7t(JZwwP3aO7dG7cmM`aTqfRP@Pl8OZj-heM&v-0Q> zBxCZ>PHb2{?#8#RzcRoKN4cZy9N;+-Z;eQX_&q7xh8g;pN{*m8OStgx=e#`my?4M4 z;ZZ*o1yMHCVRveR4VnvQ{LOA&psE=CnbfFiG5P-P$=TWo@?oEsKobrB4?O5#M$BD2 zaJxmL65zyU?b+i%a4LKUIg-WQwD#_DPjvA7R;ZM=zgT%DbV#+s1iS&f&bt2e5k0nm|* zR{Vx8MeVty0jl5Ea{)DkJi%OvUe!+16YAu-#uAfYiD=1kWL=|VU3C*yO8hLXAM@+nuw?hB|#|U(V{UFqJQjx_la?fSLICLX9+5hY;FYqe>&jqAY%DMiRt# z%9C%@%O!4-Jo+s+6z9k`zj&A$@8Zt4N3y*{(SwF)D~C2xCYgF)ZNHBEf|8MGb^RMLTm7 zc^OKjMVBI(h13Vt`cfiq4wQpiqErX?6Q9C+J`{RiM7dFPY#a%<#q$x)v)L6~!YQmn z)>2nUN!YUw*^ftUY?yy)r+s-T1Ke3xL4(!h zL?c=NJyR8%XymN3Uv+t6rE)3Kn~B zxe-Qe*Wpye`={3SK25+*g`1p8YjuRi32;<>T{`os@yK z^G4SYhHk?|B-xZ+yKcb>y{D?$^hd{3PmeJ>3enh%yl0jR*&m#jye~&@_XEaJBu+77 zc8`A00rf*MSnpbdZYrOXI0ttbD|5yG(xT^vN4AbiV6S>@j;8Gq-b9gZRjPxg#f=+D za*KRS#oE~AnbR6TtEZ*ef2T7HD{kA>mL&GB2vAz6L@E(<@h81?qqGWF6>>@VQK4y_ zyo#`su-3!#v7gx~HMd_0-A8pt?VdMyrpROWQu=GUOv8No!fw6%J^lmfob^3A5KD1eO)Uvt*NcD1VwgtBM^|`$f z=7s`~7_t|}SA|6^ciuQz*zrXSHrc@aiwU%zD_NT}EHQ^yem1Aw2*o1n1m~sqx`L%R zq4>V!-l1%wvBw8p+oz6v8v;>y+Gs634EYn3ZQGg%QQ7sXXgfSSuiY1}(cMuz*_q;H z>OR;54CH%rA%ACDyuLw6#XA)E#NgyRZUA=!_uY%@yVkpE0+wP(3VDigN=g{i@AFly z76y$UD8H#H{OWD7nt0MeJuaPoe4Q@QL$4q z<)DEwuJxxIN!H;-`f!?Uim~R}-^^e%n0~uw7{_l*8MkY94kSf67Zo(I)=39kWHs?| zb$&8Xs%OUjGSQzxh=F7`rDA`AZwZ!xC@rikv>-e$J~uc$#TE(qMFQxv+^_Pwh$%F% z%cjVP;kIepd!pZNClenSvjWpIb5zSl)j4M=B|0WV~oWTk9h919V$!AVv{A*glx)f+T=CugyC?hFjDGLpU z-Mm*|QWli7wv(!H*Ak9Z*zBj`D1LhS?h~3MXKwQ=cfMpnJ~%f8Y0Zg&6o_x9H&{z$ zz@Meu$z!3JX|kJ8%24HD`?A_HMPK3O&H7upV@+|lVRHWEBbR-P@A?B|Pn~^!5H*38 z`{vBiwi!puc{Mvx!YJ*d70EIKAiyG6%odQEV%B=ka-4QvysAT0>1>R}Pn|4_BL z*aPjQB~-bK64F{UHg~RitXzz>6E^bK_ABAL5PXD7YEs%#O6kgYyp{$URL|TWBRfj1 z=$yJo^H~;qAqahO#|X{mkDpaJJ%+!rHu~|h4P+F-=a}uQ4b-JDSUl!4PtB@^$J;ex zOEV%)CaL^`*EB^0&N4$&0_2!Vb(Zs*K&+^%?bV2N_Pr2NHyYYT5ZXH7b%`4lheBiL zb*8N|SuB5VK(A_Uxp(IybO=IlWNpc;o5J1|$UiwgxzwB7Ud$M-tVieTBDWdhTFWNR ziv9Jq>OoBpPC>3iPNM)rAt&c4^jWL%%q+ zkz>+ux~dULlaziK&-+ItpG?AQ-KHT2 z^?dj%J$@p6QW%Ma-?9%wo=X+aL|rOfohVtk-7hAK3JJYfX$dgxxX4r%338Z4(psTI z;P?bevGcr8Zm{!p=hWgt`7W}F^f@0#sgrZIj97GyGAzyDzB9h5m*`r{+eJ_ViF`0u zphvw=9$ev12(2GUr;c%zVUsqeRWx-YX?uZtuFHs`{o9JARw4K?9|}_CD-n_EJlG;&=Z?$gM)h z2ojw})j7H~oR%G>aBvont6_#SRkE{ye8Pg$Z9R{XeP*+WuY@&L?Yu+T`S;wJVtD;$ zP+X&Ui9aryr-a}BoXD(MaOraygr_vGDVd%GsW1`Mk9g_i{yD0I32G;{_EZW@2T>h} zus=+22?nDJ-7v1Q4GrlU-K4T6nk_Nrn%{DlG_@=3j6(n^;&TxjgFG<+F#k2XY9eyvBY(PdAi+VA^RxiI)Y zlI~||7b<1~0j~fPKr$=!bTcaoYhv0$Z9#onsOah9U!B681T(`2C=m6y78R9eGC6NP zw*JTKdacbN1_zk+z8L+;+pLopEE#IK&GNp~*q8V!wVP#+UmOw7YfUzXTX>ub28VuW zD81kRVy@WiJX5>zrldWHanh+uJA7=%a&^n@AcYCH?JH z+Ho_YsF)9tlWzgq(-z6;=D&b*%c$qj4~Z4-66AcGIXC6E`-^FULJUGw42i##3e!TK z-)AK!)^0-HnmuJcRPM{)+E%R>%*h5h5{(i?^9MskD<8@Y(_f_- zZ+^aY!I)?M0BXGfQxeKVk?-6Rc=_z8(o8Z5N(j72e(H8$R7v=6UIjIc4X4!Qs1~dP zDoVO6Fu51E(GP|(T&((O!o)LEMd3=uPrf^WC|1oRqXuY5$M1PYaeXb(qf<2Od8+eZ zTu~ZEus=}e-K%W6p$t(A{5X+SOW3w zzg0FMY;J<54Hx%3aDi&XIJ{*Xzz(^^I7+pi(5RE8kF4C#TIjQ0QzQWwX)Etb1 z=L_V;ZYZcd{9_^-7lg+M#)$Ym&=NhKcH5pqr|yZ**T}A!ZCfn^lGe|Yb}~eQP@Uq8 zk73JSb@sKq)dav7ch=dI)OL7zQr30CfBM;0W-_>v{|zehxE)+{NNHq$Y%6B_56^{U z%pP@bcn9IB!qK=)DUBXNy4PL7D;k^MzDJ5!diFB&&Clbg>dz}5ytLZu@Z2sgPwlM* zc9#^It|oiMOQ)%GZY~`W{BWz2r_If4$l;#zHC3-q?p*@R&Ik5xItF)^y~dOZ>zf2m zt&jdY%}0zPh7A>#h86YhZYHeDSlJiZXNq^@#77Pu9q>rwMNqGpZ}>N!6C&=a;(Z$1^Qn9I&M?klZnm=Fp=vR6i)2~)Hw%lDcwlRm|&z|ReC?S1)5ES*4`EF#3BiC&uDtjbewD=!yq zVhd9?ehuaN`LT;r=24F>Fr^m8RbLF`ZcZ+yHw?rq%!#UNu{D~YM%1lS`9hAX-ReW+ z#@?vKCDte3D8EMILf*Z-{N3)C)=~Vfr8D(q~xhB>LQ?S(=od!3|3j2Ihz7gQ+Fh$Ia)H>Kd1!%dY$D{!|l$i_Z7I zxoge@q}EzaQ^Wn$Lt8KJbj@IETb{cUy6rE6r_=WrVI^){S0aAmN9WTqq;5r?E$v!t zQ;Lt}1xl5%#!?pardiJWK%#rT#LFuuRPFt?GE=-Rfz_imHDJ&dwa9+6jl~S~J>Hbq zM-0ps73Vppw!LJgQU}YplfAY;dW(>bWv8SLgOB;^I}m#1!2YMBG>S$!2C~K;R6;gt zEp2fB#18!ot#M6T5ReiLN}IWmE+_$Am%4aiNjl(}s*>BnaPvDtuAg+`Si(@*h6ncr zLHq7r(sqnXMORbLegE(S0e`9lK}x8GPV;G+#YNWo?Nfh?jrInU_ayMKQgzeyGHPiH z(winwC@%Mws73yYJKa4Z$WIZu*{IwKp zLiv);m?6*9lR;~RgO~W&jJmxm(G(R}*&kttTn%--Mph}Bf)OD*Gd-OgmKv%WhAZdp6SxG3 zIKLRsgW2v55M8C;>Fwjx7Mq>#jc9k$iSv>NcV<~0jgzhgb?^!sTvrEF#?7U3Ez*&EUer*^6SS!y zBWi`eR1BLe>ylO$h)I-`>?ZV#B`b~#($RJq<* z!avL4FMiEZa<>t zWVSJj2l}yqquMk@nRS3Zrg?C=-u$s`;Umae>3OpyUb!zY-0s2~A>7JY69+LzA$SY2 zjOI)Gvo2yeu)R+15Vc1+%~GZXF7bDp0@1cav@R*_>NvbzTk|Uii$CW9k~P1JKlg}w z%i%Ek-cUWl-RWSjPOAU-A>jVI$uW!V3EXhUdNt_r#QuQ=GNo*SvM*7pk#_pE3yso&}Gr8IX)q~d`2A&9*<9eh} z#ITcx*-Bj9Kyy@Ci%gv4TkGn;4ILcJcGzJ7x=om_m>^6(&)yI(8ram8ImG;{2tj70 zC-Lo%-Vs3uqCH>EkZ;eiIWQDNn#`~?pe^$kQf#f?rn<9iCKq_WE0zk@pVe@mJ1WOt zqz7a6(V`J?Us4MKdP^!h)O917@RgL6>x|3@6ysM=*(TPWH}7LyMjQLsRCm7KUM*)= zTev#dmzKZcUxyI9JQm|DS}!^Rz=(2t?>cDaOuAu_bbTt|XP22(5-)Fu4>qro)vuI190r=#;_*=E>&C$BqhtUDnBO|9<@w!>aJ5e?Tp9Iq$riLwVWmi$&FTIb|qd+_C zKUIQ`Tp-zpWNm98)!~AMQSSW|cXqd}Y!M|@ZDDJ^uuIia;t!fVuC%`f@V3FfTba3) zuTVoR9)h`}qh$Z?oWucT=RhcVURog3l;lrX+xbsz!H%+%5 zQV8O*JH^9_jT8kWMuvu~JN%H3qdxl)vvKiPTgD`F*dvEiyOO;i6zTw# z<jbzLvbZ;kqlHD}m1(NrO1&~dG&kJ*w_$az^oTvs|z5F>8%NPLKVyrOcnY*w#Bq|a&6rOmm2LPj=^ zn{R&goIuZA)&8sIvc@-lx07$hCxzj)waM0dTyFEF3hQszBk_9g47PUYmB2mnkoCXZ+`>rsJl;0R9)% zX>Y6;dSAp8Lrzwolg2oZZ(+lq8fzl=Ux47V8EAR@yqZTZgh~`^VbE z6F$yXiI#=VP-VULq3gFxq{rpx*xlJF%WCe;1b#pV#ar(O&9kDse1%GUr>70t^7qa; zs=0fQ;yL<5^h=|e6H*y@Z}<*!#1`Tv&(AcxHJTgMt4e0g9t>`@3flTD%EYkA8s%mt zQhyRo^t!6L3w?$(Iw(%1xu}8)6e#V3)l#t#6Q`~3RU#isd>#jR124g?_(Fb1ul1G< zo5$-D=tsIUQ&&B;A|AU`Q@=xeH2%;PtrdY6GV4x*L3Z$Q@KV-`t%D0_vGGy{hS%df zm!J~uJ}rhp`nR#1+wK zPD+dKeuh?Su+Io}EZv1!NcntwM%&p4$rxCc5UXwCP&-#YU8hJBd_ZJWs+`>?8&|SV8i@v;?1p17lskwco1Q zvgTE667!UBez|w3J>}+2@v(S&`C#`j!#(IDx& zWy?HGdD0PMRy4?r5;c1Ar~wA__?=zDO{8M(fO4^ zbhNO$q#Y%z+&|Q;5}9#kiGYy@b1k=$8R-5vw~&bW%)dM^pf?;|;D`m1O?$8V^~In! zX5w!!?>ATwMEvm9b;Jj^*=zDxIBkqcmoB~{U-|K~#b)R?0htZ%eb_`foPD4W$v_v^ltR8;^k z;`>l)pZBG?ml|c@c<5A?4Ct^$P#hAC(z+zBl(`W7Diin z7M=1As-%*zHpdk|M^2(=F+*4BaEY!6xfOJE1Fi5belzd)$YdsP?e5kahzR&SqHD_( zzXtGW+z}AX;E>{Tz;r00ug?ZovK)O#j89mn>c~ky6ji4m0q&zd0i2ECpC8^RHXS~W zZ!dVg2tU1h!Yqbm#<)w+&!H;ScKR2>=N2T7!fw}aE74#w+Ld*b6g6oQRZ6U{T z3Qt%>!pnP+k0{GZ08VOIrIjZoo<)t$D6JSbrKTIxdd@C~W-mS&9fg*bDl<#PQ%49h zj%gdJh=uPkvwUsfd9K4ohA)yFUT*>c>Kb5J2!2rP&BPVR5sVhc8KVsjl8_jy2^Jp6 zNG$w5ipLq7FLExvCEI8PF^-!-s*O&My7UnUbX`U}ry^gOgK>0RKlj6f>Ive!K>7Ov zTq^au3}&F_i!txn=WN)Y`iOGc4w^sIp=tTS<^xVXh)}`0RnLW3G1nHtkLksNHh8s_ z@g7VPdZTy3sbxbHet(U{8zP;%Vk$@36@yX__yM_P%qWSNreQvh^)FEMp z69g#$FcSb)oms`ND&Q{`LD-_$t`pbl3+>sJ(p^<<#u)~1b$M(rs_Tq#8_vyl^NIC^ff3V+M0(QJJikII1HBR2B|R1 zc=h;f6QvB{+lO=SlT)Z zmiH>z#Aq!aKi7riTPLZ?_5MrjNHyA!f67E9T6XP-9ekPr;cG(0_5}kRxYx7v0R&nL zy`QnoG1U7$^ZN=UYb)0Gx0PDJ+3RFExe3-BMqFPkU~u)J5^zZ+qHz8OcwbtAV1#vnI#m zF6dnsCn`}S-QV3{oV1^o-Q!Fw{rNJ*}s1Es)8l{40=6Wyta?Y2-VKc_8Td(+n1BW%OyNRK>eSUYd0mg7x^`IME zyKK6h=iK+5K%H|S7~UtbE!ldONiR7eG`}Tzu|a5>wN!`p;L=p!iHec>dC$sd}#-RrPw0X;)Iu z7ktCst;gOSZj;iJ)&2sG8Ux~SI|^EHZViD^+v%7wcXN_((6e9h@rbE4_IC)2C%y-g za{MqVd+PMpC{1!5&@rD05D1u$>81RhP3?{yZ4DUyi-s6T_IDVwK?(S+O5xj_1LkOc zMKBB*lj+vCX~D1m7=8B7r{JW9xBiuhd>3udp@26U`>S|qgAUWHi;`ycbE3kL4;b2W zXl}~#&0x4jr5r>TI~o<+&w&6(==+5kWfqPE%!_ia%JPv!H39gK(rN$3Voa2htb4)B zKy_IpP*qIHyCl2dSOrF?{1}!<`R4fF4Up=(38z^3A%J|w-c{q}cd$M|{^OK5OE~#} zB%--S^xVxe{!gGE&^BY!kw%$@c++)s6oAJCfvzGhT$_-{; zCKCs;A5A6;#^TsFnJ7Clm~5_#PRWYk^L$cAE)!c`=84I_`xngbuy5dtJOp;Z8Jc9diBHl!@(lBdmBRjXpulGpzbtx;(q&UOX zt*X1%W$~BGcAQ;c&J#@DV~sq^WxDiLoF`n)!JX#dn~&tj4Md?z|W%+R2>g``RYC0yBxt%qG|H1rpor$VJoaP^_&a2Rw}%E8BZ4e>sh(pqCSi;?Gl;vinN4})n;&)K%0d_@ZhU|>hK3Km!DG`+ z%f!)$<`ahoMK^bDG?F(rnVFdZtd9yEvZ+4y(Y>xj36`yN!}lwt3DWt6_i@HujsgCZ zd@%ymPUj)`smfd3rZOTS{vw{YXRfqBYB>~F7ABz>Tf=!Z64SvW5LcD z7%`MQ&O>bAddW(^H&HFxhhsBdL$`qDgs9b@^yvV70z;+|#JeRZpZv3GzlrlhP0v8L zPJUKdIa8wg$(anDqk6sht42qu^3;&U!|6oPj6?vjt9kZ|RGPj^p44}3+pIM95|x?p z@ImO$4mimG$~CvoL&Fz@GWX)C1$F{ za0eGZ8vM{a)I5yXwlPoY?0nC*D+iLa$^5V$2U@gYjsO=*RKdcy9!G9(2NUZ`wPNzXluBxx^0MH91r?$KBOb9F&cm7~3(go?9}t(0t7=r5PPZ z(XSBz#VKmnpU*V78JWv𝔪hJX)CF1}+vmD6X+dL2~2fzNx;(|76w!%VSP8)%T6d zn*Di^>d1?xh^;nw2SvZT?RNz~1D3PM9P6HGHr>vUh94d@WQqZF62y^42nx|Nl`BLD zP6;cb(%0~!OpK(!Y7DNZay}}kT!aWV*j5_SFcoWPZ22uyuhBt_EkRZqX|7g@wsE#g z)xZs27ZakWNj2ajbL*D`A}0uc7j=xPq&-(y1!FQV+29ehy0W#^4N){OyLv>ZWTCdQ z(qVaB2Y%a@>{O-1jZv=3pjl&Tv_$Om5w{S5Ua1DGTcckcxDY@TNt#olMP)*SR5D3H zwciC9`}L!sS17=sf}rdzLAJx-M9n^>F;WgbXa=6*CFQ|S##fN310pljpH1lg>Zjyf z$pc%-Y0iosrV_=@?KQ7I7`HUHGDi+%9m}mT zTdY`A;b>|CmY>(J`CAFJaiomEIRV(BqodjS>-}R$cc0mEOJNnaFzW zzh38)_Y3m8Ad(OCvV?=?e!}k7V%GgH?hzB=|C2Js!pi<{B8qn0hIIfT>d>VZ41H6C z=+80>eW@}l6o!Y`7GB>V0~6-hoQ|t=NZd7Y5`mjrpC5-2&1DD;7beI)%kpfD@l{Ks zy#Sp`F#F(N)g*&qC;RORPouf4@@E<8MrprGoY+O(WW?W&=gk^n-B!!5;ilx@tkITr zbcg3$7C5Y{2H$$WxswB^Ve)bu*gF~zD)D^#d#n^`iXU(5?D!BOst3Mp`(YWAh=IqO zku9s}5{{N?f{(~rqam5z%bI-On9O`LvtpFZ;4Jje0OJDRQ6?0X=O#4rL-qp-Nyh#V z_bA9OCgq0MfnzNoHFea4K*szXfdnST1uiK+CP$+vwjSct@7z1r*h{=*UZ-glqm!@= zqw!mrAqDTCj}cQh9<{0Woe1-z-M`RVcO#?}FtpJ(qJm+NwKp`fH?uY& zr2MaHUk4SPY;7%#to}iO7{4AL7zR~B&Ht2@{^=tNBNII{8z-waAq(3VQuGBFu`qt| zK3_1?7Z}9B#P%Q3)IaPLfRl}hkd2**o`r*rU7L`V=^u^i3!wT!h5l9iKY^+LQ}fGr z;`l1V!c5Q31mGZK%P(M!o|BpBA4`aplkw{~Ga)M*>({ma)yjXFSO8{rdS<{^F99r!^j}gD z6Cn#H8$IA3F6*CK|JX)f4X}R|XZz~)KW%*t_n(jdW-|RNbpL}P``=Bbe+~UB%>RT> z=C{??)9^pT{6BCe77qF^|A^zi<4pe{Mg5n}#Kin{Tfc6`f5({^|Nm4bCPqejRt{Fq zFXoAjlm4qF%NIVz%t_C|!OTI($-zhuVB_HU?<&*3`u(3{{4bS>iH(u@|DrN+vU2?U zE9l(AOif8_@ru*sGU$K}01!_IMi&r~M-(ssqG8Kv*?~mS5fUTHBMMKam^Sw4Iis+S zVb+^RN@y5}j!Qk1*xjV1NFhr{t{KkIcB+#{FwMSamUaGRPJ8s~Jbb<7ow~bs%+z!$ zJDFc8SN{2}rW*$F_fIb&qxWKmLF@0KFF4;WwB`7^ZiYHpRKCHj|Ip**>uN95;VB2s zsliRyFDxo2thuZ_eDC-CLAasD>mxrlkMYnUw7c?+yrim%7Qttu0O?#7jQ7}SaxME9 zuK519t@3$XOZi~*#J3|I_`jGeO`YZ+1_(ULu!x-mKR;UauZdNirh2MDIaFuUSf~$@ zmuQZ&=Rs4h;H+7eDrwEH2@STyO!Y3jx9=`*(x*@ACnd2^#-E@JYa&>LazfJIJVhQ#wlD?A|W$hV95A`72Cr9%J}2@ zQ$SghH)rOA=^^EtI6(U({&h<;K(_IvE=1THasE@tt`~I&E|uV+;KAK9S0hSa_=v7X z0v7~Xh+Fe(OVNk!{`32fmXuX^M)_6Ib98h5d?tzp-D%7j@h-m#(dpJp#0=7Abn;oZ zB`C;%V)b7J#yF6z*qpkErU<`lC;U)N-@9OTB!-~^#}pdA z@9OjTWNNgDf>G^wFr1`)$a!t)N0>b2Iu2&6DX$9z|^83D~1Inh_PRV2n9dt#X{CtyX8tEWi*1W=v&&N4MYIT?B= z_veKVPBMzH@`kcrsT<5|zu8950taQ|2^@F14Bi-m(XCA!>92`6q-y>+HfXe96(Nt4 zV)pD!8jcwq?wfRGzf!HxHDt%~-|K($Tc%Cc)LF$1_^lHR z>u|5pb2^Q{+J~vu8La<&g11NRXYLqf2VbIc>T%MyF@HriL>z~zLJ9Y8sr!K(!^slm zDr6zQu1V4bhZ00|g&4tF2Z0^KItSI%t5tOxCj5&@Y!yt^{zJyMU_7vPwrzeBYzaol zf2oG=n3eL6V~XtCCKYNA=qyBdKt;?zzb28o1|BwsZb6c1!98RATqTJobiZVa9y-Fz zvN@l3z!On}N6yc65eN34<-{`{C`Xdm>jitgu3U4-i&Mud9QP6kJ=gKro7n~HU*fh zY|dv>=!ji6q@Cq)J>}I8o+Cal0nn{`3$(%QO^WDr}fj{g`N?f56CReXDaR z_Hk}VcwKJU3mL{YWeDeIQ1JALS*4r5$bH~7m;J-fUf-NARmkHOduR1oBi=A(Z4Ifx_;o6)~b=TMG>m!*!QTcprw zHHisx<>h5E)o$!m9c_|qCjL3PNS&wQEOg24EPa0|hR00FVxn}gReAW6tNNqO{0VAm zSe~u#lga)uAFLD zO|ISlN8DXD#TB;gf=)t$OK^903$DSP;O^46ySqCfI0SchcZbGl+}#>)oSmok%+!A8 z-7_`c<_~nOuC?x}Yjq#zbt=v89{!<}t z^j6>U-pXh?yY%1g3xDr?-$9D!|DTKT9p_atb2o1$8>XLH?7aP65d2qWyKjej-Y<$$ z4Lt9gKwBQppi2S!i?^4xk0x-oJERBZ<8gZ+9+}|vFwJyjBv{xNl%?RftVk+VyVk}Y<@($O47Ai)`%Fjy1N>`_296#6{`7P2cGDSL` zLLmDNQs|!C{i^?{|E32Etrd!lZvkfSw_AN7x$tGb@C|=V3rUjCpqNU!Jgv(giEnf- zoH%j1?wZ#VZGZ4P-+Jg?U0zf09&x)62!GhLIY-)>=rLizc|{~S4)_`nK={{0a-=z~ zO74wNmRFlMgAdDn-}hk@kUoM~!0k3s$1&p1=QeRo9Ov&}dC*F{?XkxCa_u*wH{wmu zb$I-~cUmVZxkT`^GfEdgjt_@LsO3?3C$*aRId}8pWFyxHauTE{yUcq-zy7H0@hz9= zOB@PSP1i9wKOH}<%}2|oJT}B7b{#!PHm35s-Qg&f2clh=j>GrLz~n3HRw(tp+#OmE z0-!Lyl>>TZ7WcX>@lrCrpS>3Qp|^nrpKhBe4a#1Wvh$mN zxcg3zA#o>^L7(m;%y~Fuo2N8rYuoM-Rxki153f7Wen-6(+RBhWHHFQ8YW$9^-3UM;dTq70!D<4I47J4RAkF`-Ev2y5C^+65%laC4tQ+ zfvf)uOV`fwnMT=&to@rji;<#SzkV7++BXC;_$wZtK-y*e7ej+Sx)3-{Qr($v9p6dC zjOiW*-qWbKLug2~9pG6-7+x`9wuR+J5FgPLU^li&2*O3LXbHkhu2>0%cDTO?aHB~Y zA=~up-Wd}entYBMX4Q-^aU_Tvf#XJ{HbPYxrsc*g=^N0Dq2wk6Y%g-7`*I`5)*((f za;k3=R1UUjhA!4(SzqC0!Jrs30!O}ShTpe-$FD`jHbQ3_Hn-`ctc@+d-V`BzL^u!N z3Pr6Y)DO^eAn!!3nju9Dc-iUMLGt*r8LqLz@uLsyfYqDKQ{*st9Q4X>ZwD+*%t0Nf z!E-y9Azjf5W}knC$jYCjEX-%)5bxEV?HFM-NULCKp?wQct>LSgJOHa5rh2s@z(3cy zyYsmI&IKtQx+`gHhW_x=z@>K9lR6ET5?*tl;q*#p zFrhZ!5vEpMRaVoyu~a>9Uf;dMbIEhSF8gZCBX5P?WoujI=WXC`?_S*lnVU8_9!+^y zb*~0(f#jnxv0QBf{CK3@J68RmjW>Ny*ATqNV~HrGKX7rz_uB~avR#d51w8KAo$VX7jFozlZGq_T(4{n-M4#IFut=p(m&O>6h z>#Blv9j5`kBx%6ME6Jk-WiQV)nZwEZk<_6n(wD3=x=7h8@8aag;tpB55_ZNmSAVcc zqu!UT#OeG7J+TexNrh@E%W5=Ig)CsBIAJyavr{~t1Qc&sbPro>9Nr;a-mfKTi4GhL z#12D3FVtiw_~nGr;zNuu|5=aa&5jftq)cZ8JI?obb@4-K&sk^kSCZ8nJpo{K+9q?VlTEO zs(ad)+FkqxwFmevM@2)HCdQIsNcL0yviA`!$T-OI>mD4{_Zad~@+-8En!y8Bj}ov^ z?%$DZrQQR1cTR!zJKcGDc`zrTt+F0cYv*U#=wAk4ErXu->U;d9o_2l5DxzAiA*)%H zcz3<{dr(!Y-GJ-+S)3j9?)BxZAA{!KyGsOKfjYJ`VYE)TE);GFG z9C12!T7II79kR222fBlfqx$zB*xqVSQ zWHFl|8a?9@uh(FyK2>R%kuRO`H&A`K8af*OZFR6hwwY{`b%R(5HdIC8;_KixV+*8f zAAsS;d^aM`bX{{I*fo=@0PC`ahg1-g`gx-SA`Y)qQ0rhkrXKt3Y#z31(dGBJRLNi; zOONq7UVLOn`00MSWZ+XoReYI!=XL|lf^MhxUVe9_TalOECS2WDWZKwE)$ja2uNZ?r zsny(kE-wn7xrnsWx)CZpEtI~|G3?gudiH9^C=)s597DlP%Pj*88B$sc4;Ii)@Qph%|E$ z!#YxJrjm6D!#Y|GO)fvk9QQ_*RWaX(ZgIlO+u_4aPXiu~r)|UPVD$Y0OBVKD7DX;( zk^YG-sFLxw{I?(ySvoUgtpHSViDmMz!tA*xOXOVRG6f+_*8TEEoNV3;(Xg4T?_2%K zx%S%u%EW)3(Zsx1pb}SAZoeG11r~_>3}&e|q3wXWM+R8lSU4Jqe2wf#2w_IKk>d1MN`RjI>;lWDLnZ^ruC@ki8R;Mls&V z>~^rC6_D9SVyGe&V6@knB!3$+mF*UnDqidPRK92_-)ydRTQ#&qXWgLIz>8_$O?WX& ztIklTs-w+7&v?dAKcp?^{$_47)YOXVzD>j*Ks_^gLtGge>r#@+5%D>&O7hf2({|S) z+_G_M|GTss1?j(Tl=zT6S;z{zkKlsPO*Yx`#cdrh5b78~#`lN_jpXsy#eo7wJ^?E3 ziIxIo-#0fxw7DYuBg~%Wy|Z`&j~2_Zfk5B8e>1`IStUPM*wJu{YwK%w7v!zvO8+ho z+rY4Aw)%5Ib+OFgS9Jd1&{qZ|wW%^nYc1`%rOGBN{3e0b&le%)v!dk+F1Gg({?5ny1u&#H_Yzow>^}L& zJtrW+{J>$(hG6|LT|nJv6(bq7>9TZ~*QCMt)})Be0{ZjMme2#&iDD!g&3P-EM58`%i`j4@@ofK8{a{Pzu$*O7tHkQYrGo!=CHeOpr|{H z+qcU>Z-;S|n^K6EB0RzK!CCKE-7)W+l8_tSJ1+HnL*x@n@}12SQmrt?f`?NVUDOlr zp=3TNL?nc(O?JB;E3B=28zU){fz)7`oZ!3a_k8h{Q&{#DiOea~Ad^Oh5dQe}=tx2G zjU1!tZTMrHhd?mKExR2@;WzJT#8iwTYKo|b%d;E-^$lk1S{7zr)f%E z%>qQN9WCEM*P-inPgl90% z)}A3dX;ZGI<2x|rkp-rjCG6d44w*JYCl;U{EYDK?bx!xFG(-(_%$glyYr)+ri(l3X z`<;Y`R;$6rnxl}lMi>=k;h^A!3W52f>nXoIVsXwPdi)`bZYR)D_7`MgP3+qozb#s{ zN9qaiFx-tLE^=Ftf%(EQKB;3>l1Tb1g|SQ0ty-j+T3Ro5&T7)^rV33vG`1i4=jXn0 zPZBEgMC^rIJ-;+Y}z z>eE-OICbFtYT>ju_fOXhAXxKsD(Ln{H#_-H@cQ^v#(@O|3|`mGhpR};CoSU$2|Ec7YpCZ1G_Vpe{gkg z;yeTim~r2~+N)Rntqv+OpS`sH5M{kJTl0wEKC^6QlD`OU=xV zvn2zR!Ij~jmIBlz`6*a*TqtF~uPVb15g#WjVkRA0*~K6ki!uJgv&Ha9oPQGj06%&@ z$&9wSvgmhyjD1s0lYJw&7F$MvLIm(V+|r;-2UDK&C_wHU2t5S*Tl!l5a@e%~!L($(PfcSW%tM z^HJA6AGpVjG4Mg>cLN$@wngOUa@Md@loLKT{P@bS4@F54FSUC_t>|4*uf9?m@vDMC zC8TVY1}lV=ie4)!p@c*ZNs`on?$6i#)uja-4)G~|7veNzrA zAN8-i6q+=vlrz<&Ifl5E2}GI-hDf>7%D)Q#&Jy2IP8>tCi-^}PUTB$a`SpFP-a-yZ*rJG8jeDfwnS8wxaMR_prmOzz`7L^yAd9)2yCAPQwbd~EQHB+V|sqVlTT{u#QLW%)!a$#|-GF7QmK z%--DK=R(*BqCGBC`#n9`?`zt!m9lSTjAhDJHcOBD^tA`@oyOc0u_*Z5-IQO*t7TB@ zL(zT~zLs~u6uCGLkBA}B$wW@aF-I^**2JamMZXy4j%hoxiQJghSLHY6i7Q!hM)Q=j zhq;_DjU@;kU)9Z-x`!yuhW3xbl(^lxv}fKRgOPEu8>&Vtf2~vPId!nXAOZ6&M1V(= zPUW*o$ZU=KrfhuhWWid0Q`aLC*@v-ZuF`Z%l>hASBfD7l>H`>?phAIDYh#r+?Yne}o*#p?5L z^EFgRa*5{As;1E_nUkE0bGQb;2|b2-ie;Yp@(oy)StU2e<$9ihns|h-<6K|?vUQ!= zq*(>tmS2vtf=>;3sW{gaWcJ;=!%01-Ox{Wx)c*M-o z0XFfL4A)S~qIk``sb+mC?Hz9uFR2W0`U(c*@Ce!#4cXWV)?2&8>Rqi4J2o!~? zV}0s%ZP!1e>&uXa@lHm`b1WE1?FWZq(~=KZ(vWu)^PKTQ(l6eQEpbbL9*P~dCmgCH{ zfK!pP(x5|1e^x9B3vH#KlCE&ep>;7t&NjmCP>$d>9)7NYDq+1?Tf_4b@}eqv$f0Jb zswQDUH&7h8lrj&)RpAFbOA{meUnKT6|NK7J!=3ERH^Lsa$QxueQGc*DXu659R&F!Z ztiFc4MwB^LoHk6mx2mS9R+3R^{x44ecdwzQHsBfXl?6duU0!KQo!sYQZ3N*nvGdXf zM#q>5cf%xqz9v7K-(ujw$BVO6Zz-nmOET&xe-@>kc(xxQAOZ>rztDy(eKNsV93fPA-Y<$b!?Uk&z>Y;x&Y0>R$(9M7~OiM!zK@7E5XB6YfrzAi1C;@HxDURLr zauM^_IlBIw72uK|u$w^c;Ri#5b50gDVyA6<+!upGYmU?)L~1SF@7*OEa^+C8@2?~p zNi`{+636G=>58D-CYQIByfVBo4c5qFM@EN?>Z$F*$rdaMsmRGV3=)1hbB=0bx*#&v zelof|8DE?pNidas$!uWQ$-wbNEqE#rifkhtH(l$MbG%+w9V7ntG=q*CwD&hb+B8-Z zpKEgb<_Kqq9~=7>q${vHwg{@{{JLMlE(HUft?)+_ket7(Now%Qk$0%j`H#k=t>QP1 zer1Y4t$GW1l({3x!Z;%e6`tf%y437yeugzOC3@sBA4ea<#q?|ya3oJK`?Kd2r;+5y zq`OQQ{D|Myv>OV7Uxn%XrTXK7bCZmO1edm2J_4H}`bsm_n#>y2+RruVHOoxg@)Rs_ zL!oHcQn>_1^h!{$Pgh}izA2{(dW1Y01DEu~$()guQ__x=8pWkRMq&CBv%s*TV9sYm zbZD`WdVBA$FU?`Jp|lLFGp?q2=K2w*B9YZTVx_cB$BlWJ92Fc-1}+KkSUIP!Z{Y%> z#`DVpDP8U%$GihlWr}X(?X09|>?~}eUgxL1)PbVxP)(&o!e-E6xvxW{>+^L1QKk4H z2ZSIvu0&JQDBECZHlwRbfX$zPQAm?1g1HVn10@zJ zgOPHgMZwi*yh0I~Gb%5;*1FyM0Mf_mh~Ec%o1#XU6hMu_ok~q)tkOZ@`?$)N9)WN; zbSNEKss}O5A;w!fAs_b@J4dk*%>JWSFuT*Bpx(!f9k z`NH2o*@SYxK1cdJl{4$@!Az{(so{4>gu2Z(+TCpRZYyw!_1LvJ?kDlse%XfMHicqU z)E@G1{6j<_x{x;gNI~gV;0vN|(0kZ=KF^yz6yR>!k^j%EwGa;XtaTpyFzZ{!E>hN= zbrR2J_O;1z9pB!_LgwCeR(F3d?wds%#Ppe?36J{(^doogjr)xq8hEV1uFIzgT=jHX z0`3{`#rvOm-Ywl8R~?OQS&a&nRQpMb%B$UzX~>A%gL+9tp*w~bRevmSEKTX{kC$1O z)>zXRsH(L5?Xfnst%_FRY{#R+r^mjh`Sz;fm+9^m0X*gfl~rpgU=;u zoImF1ssrEV=Q|{Zm6WsGhA1>9Lww%8F z4?J!6F-p(Y1^5Ds#s?J21!>k;=Xs0sUf>$>+{qIp-A2Es+S8vRRyLb11avZ{@_P_$h^iCA^npdl8-6!=VX7zKC=8%f80p+%$yO3gX6eFFjyi9A_||9)j~DEDXPo7%a&#$WraH1@hDVdrVF~BofsQ2%c@gZiVYhSp*F<051A^` z!t4yU^z6g$@eAsk~>p-Nq%c*|#gQTC?W}cfI+BqjA<9&l9m6*6j>z6cA`pzu0x1 z@|e0CrdY0=mP-p02=wJebSn~7|1tuFSDRWo+g2-&PWp~Er{5y7f!=e9ZmPusLm_2Ki-hl7(P2-afSc1qN%W>mX ziSsS$jvd7>FQtd9d_qNG&pwryd0V|FO)^_3pYLPPzc(F&87Z~;x6?7%Z${6;ib4^~ zQQ}J81Ais0{Wp{>%zE@~VeNx>i)ASR>GSwP-aisr3^XOty&s>)XIMu5Hz4sprtSZr zToRTxE@n>uNTQ9AiAJoEqbT_o(R|CMh2)1y`)EZ;-Hw-{~f`V8;l7eTPm z7eUx>dCe%>RC}DiOd?hgwz?gMH$o2_+fOP!Oj!S_(0}3a(5Bfk_uU_Usf-#Z z_jHA?FgIN2@j^?!;2#BV=J6xka4YP26{(U$hLnc8nnmVc7*KcDj-kjnVa zn*Lwtk%jC3vv=g?`bQl(esGYma&dBz@UZ{*FShZ2IS~J+ZRBGA=Su(Gj{leQ!N$tT z$^Bn^^?4VJhuTnk>qnKAH`o+xT5>Kd#cU%*lLy(17XE~Z`BhcS<`@-I0PC0PdDL>; zzk;Tp@!$GP_@W5tH9w;wJTOtt1&92;_XaGjl8W7VJ2qWZEUs$}K zj)Z6kLsWpRSNjF}?`vwmg~A>~k^$LH=0{4@%xF(*47(L2xk4SIOp2{RH%F*o-nU81 zhGKAsb>`^0SmJrIc*{n{>y`VFBH#VvLU+okmp3}?za7F{{=~@~Tp&Nf#}$2%A4*3{ zC(j=bukA=qHrp!I(}=I1^>uV5&)uhhSJmB8N#*);bii3H*D->fcyBl!WXVGQZ#=t!`GUA4L-UOtzGQMz*eN z(7^lylqBC7^QwLNGEs;GnvX0wb~-oPl{|f6lSWYnip?K7tv)1Lwp@BcUsy;lOPG^8 z$1-r)P6v}*DC~7(!%=?+8rsakTHkcBPLTEFa=Pt!Ew497vX!>bvJooNu!Ei@6p~s@ zVQ-vLn5#PLa)X>DtE0XbzGJQNv4dy^{H(C7K*6M z6+0OqoXGlq*ZWlaYP+B3P4q^OOh5v7F(v+V7L1;)%CvEOn~u*=%sbh$4JMZX zyd2eqc8cZx?ARxRTIAY__pSJP-&DAGy%Vx_$1suRK$)(Y5BOSPOaxNacEinyVg`*& zn8zHYpgwzwGvyj2#bz4}=0$7un#?V!yCM8|A?K#$**$PWm!88pd+a&g&ZWBuKA~pPbA`@5hqd`s!Lpb$!*iZ< zqJGWeHern4kKD5nMaT49D&u)shgxt9VkYHmBO_?Gk#_;-N{rLsd@|zkY%=_9E#>5V zcFzPmk?1d22)|3Zt^@xiU3qI7c86{FKxXOPb@&fvsY>CDSB~plZJo292>@&SyGvW} zCe-T&#Z*i0@|3O{ISBYiIt+}{&r8I{Yq$ja6s%2+f83v&L4FzgL*G89i}rfyr*iPU z;V_uX3;X&xw3^Gt`CPCcm(C@DiL07go#+HeeSeV-4@U}qa-0pDh#l+q^c@l()K$MFQt>PGAx(xjrzC2}LR`M(Or zOFXFp@DkzO3(a}&ke56k^D3v=MKXiE|J1FGQj%r-W`%lMSGB5%3vi9=>;=4n&Zn4f z#c(jVDBeipsB8D;Pm2-$RyRdt2GBqaK>s`rG!GF5!jJyj_b$rVycNmZ6K992Gz7*uw)-bF+ z=3{-QSsT_Sv@cUKKAE~Qk^MG4y{c{wH1_t2QX5I5$`m}r)ANyC9U7h+$ZkPujD9!9 z&qxxQ%CLD(Gwd=QrltS0#bh{jAR>EMiDWVh_Xh``W{~~I{xs{Eal0m7MT}rddNZ5g z972ZH22BI|o6azSTPm!dFjKc)fFLo<-p-daq$91Ummf#61_MPzm+v`)UDTdCWvj&; z5$@0zXL2lqh&YW%mTz-eQo;L8hr&UwidcOclZkW|>S)%}I+f&V&1TCe#tz&eHr?J$NuG8-`q{TI0 z8YB-_G>jnSiP=VUSt6aiS~~uRI73A4_|}v6j(S^XCVK!${tu3(_fA%wXHQGq!*$2% z75n)`gi3Jkq!-FcRVx_a>=#kWQ5b)9u8-ARDGY7Nmp`vsj@VB<+M7$m-s~vAnZ`fZ zk>(?Qe22Zf&XCNU^71;o6hDkrQiru^NIZ7f9d41zbPK-}#{||ohiQ%%tiH8T?{2HU zAa!;0Z|5B-YtS)~OjwmDr?#*z#yAs9f0bmBh*v@eu$-Cs{0Y4S`bQp zV4i4C{$2Ond(f5Sx$Fq&$lka{L_b*12XC#DaAVH9{~$ooOdxpkHP*UC3Hc>>h1{(V z>tO|9oG%f7zPvUtJMBJi%!5io}S+a z`CKga{Jt`U&3toBO-~M|b>}#sX(24u*TbG$5_~6Htq-f)oNo8X@~W0#5@W7TNC4ye z49Z3biDuu8<^Fu7s(zY@t)N^J#NF_YtDCu1$hFznLc8f;>w|P+EH_RI&S%{=nA|Ud zD(-LK#{S4ZQ{%aod?`t8=Mz{9(kmbOv-lcdkTRK8SPqNpt{e%Ysa$kiEC ze~YLoGRU+9v(1Ii0lmwP#4jP}AMok&YnS9(Og`sIjXlIC1_?QG9boazfc5B2KjCNJ zW8biS9D_5hJ?o+dNJ)mX6gJ2{UtW+e<@xn@OU6yk!Ltx;Ph9xtd$0g$=u_>ttGW} zuh-&o%^xH)U%g7Cg4|W`JA&C+)88WoDp1Gm6Mp0+{+me^4xtr;7Q2M~edlCJ)ktPj zy7@V>V^f?DGk)%m!FIrYS6CN;LbI+G(N=Ihaw2m3lh z{47}IaLo@2yPIi7BN=o4LMDIYw9DVuO{Rp%+%j)nYl_($_F{x+bsVbH!l_J zFeBt{8tX`;CWJ03XJ%RjhHa5UN_3TrRh+QZ->8Df?SamJtQ5xqr zv|DuH?3RCy8~pthxvU6VlKiLu(v~3JiII+!9>OPr(*N|#yW=-XAQh@KBPZ%*e=OYk zDanN=VP}lx$9;c$7TIX9F-@ag!qbdGcY*y&QAd-KT~TGhvPM!7yumFM529yGHjM^} zeM@e#EVsr~9XK)iqT@jMK3jM2-67FSE3VG&xAUlf9LJnv;-7Z^`F-3AcLcXH_@2q+gXXq9(2*&*ZaLX`S%s-S?r&AeL}+of znN#I`#%+_@34`iD_EY)j)A!7+N$tz?MtvjxE4r#_?(qEhB}pJ&$^}D%8y|-ZPlfb^ ziZ87-mLC}5amS!mA+Fc`anuv&6?!|}QF+#^3`ie$7i|2XlYVHs zf`&96d8L$z=8A{79cc;WX)s=`>x{C`slWHdFpAiTxgApc4&=vD`08U`R)6W-J@T;m zwZT?BidjUfipSbI&WUq%&xqrWvb$IOC^FCQQ2X=Aw5JcWL89B|#_&En-RvVra=F`^ z)0>=v7*?vw!GC2e4hHD^=}6;?hz@n^oo=y^o2=c%pKW(0d8eWJvu>&F_?nrOY9(A* zrL(@AkSyigBSs_;=W_<(I`u}l=D+dK*yQIX@L!b2HR?V^RlB0>mh?G*vet^$gc8tq z%Uv*s)r3}@&URY+Z!vQNjMrk{DXqyOszPY?u__Bhh<;)vFN23qje3>qTi26%yUE_# zcP16j%6s<0uG5S2UPajho4A^t^XARrY3+SgZF}yHMea*RS2|n#tqXlXl`H;rLPP5N zWu#M9=us112Ees@T8#=PM-u-)}>q;)`G~c3aCQtyn=nDelOtV1-aY z6B74vKA_4C+qL^-+lk6BSsUX_j^m!Ms>E3Ff5 z_ngLKdE4@yw^lHqp4z&{FAPJO%dJn#PLE3T3T7y0YMtQ@=d`~a(|2mHY4U3&d$Z(n zP3W82GT>{El4eEzDmneoFxU4(@hdp7$kgt>E^OjCM)Likle}Gwa{rdR2^EA+>k8c~ zll8j@S{-g{xr7c{m$KpXX&m`mpi`>H5it2LjPF@FtF7Qh*+4Fai|BHX_)etb!}rD6 z6?@+Sj;;d<$bEAsr0x@98km#G&k2{G;BoPP>GH)~NwO_L(Aj=U ztb9{e5E}O-Z|&g#2~l0RUcR-&-1z=%+bbmS7J?^kSbq=k;f$fQNAnaqT8&@-V6-W! zrnvOQB)IcI0iCu|%MLqUGK^po6>L%&SdZV3ZWYD2R5O!(z_6i7X!)lkE>-PM4cQ@~ zhKe&cU;rkgTis9r4PS5vv~l*VVM<0l45DI6;!RAG{%tKN?pfPsS^R95vU3d4d@(xh zK+zBKpV*n4)jYg%0BVP;Y*j3|STgfzd8a%=?_CL*8_EJlJ~9Uc-N-Fp(kj5xJ*Xdf z%DlU>TzlHpJkei{Y7j@MtCsiM63@6~qH2%?Mwj1o}*QQ zhYoys&!NLg&+az;!rSB}c(kk!DD}4^oV7Ee}YFf`(J9-PWA# zx*;DcYtI;K8E_#ydk}TJOm%ewbowa5_j`!NL7c>M*IKmNU z@E$3II(aj!wO9@sk7oYw1Zhu`D6}@Svbva;v8-1{eNVo9=(<0fXa8v5sLoGRXd5qY zpsXe#0L?i!G~2(A&p+Eqn71;ho%+6e)W< z!eIDrkFQkDZCiIvEAbQ7d8lh4fXsm90D}dlJ*zH@QnG@9=SzG~))Nf?On>yM5hFlr zULC#Y8DikRM`e2u*t(JOw(mM0Zw$m4TGx^DkGj^0npJ`L3B7nr?$)3Rg6IMC5X+|m z>LYAMNQ%7;!ZrP z73A=sZ942ub-@RP&lYgc{%_0eHD17mo%H6wC|^>$B-*8jwvMU$Tssa=5c?KruJ1CA z!Ull9GF~Wj=XL61H0{!TCC|dkm%#6p=PK}byogO0&cFJ+_c{PU@Oy zWFa{GI%ii2PC%uMpD5!?A@;C>%YJuUV;A{B(^tYvyy*Gj=E(hGz|M5)O$W625R+zsJH2)wvvulaXfk!>~TJGBiOws$j6 zV9_`SKT2P85X=m7s>0+5*W`8o(GTHW)b_-eLGRfQS2l4 zZuZo=&F|zz-lYN92IMdc{Gl{!!WqbqMQi!0KM~sfZz>c@jKc<#11Oe93%lN!Mj)N~ z7A~WnY2}YYerH-vKhCGEL9l^5PqNg@Zx8=Rxi1fUgYHHx^Jxu3bA#P!UClDuD_ROw zZ)~?us84Z)=gehb@$s&Zh8^C10|)5Lp(feUdHJA+Mcod#%qNr*83CG;hla=b_+*U_ z2Kn%wgjyMQx@!aW<#~qJMv8~+;w4Ij>^DI(Oqn>|93QUlOZ*7%d}u?d?V(O_WPVKd z8!`g%hEN9No1ygzBPh-kHq=fR&XL>Ll&e(~^TTdzOp!7lotFaRGt-=3%S;$A&ibF2 z5?`1_+)6sxS2`Zr0hG7vOH4pRZwg@_oUcDWswv?7Hzb!ikI%g2-UG6aWafCL7Hn^U zq3rwAxTYHs-EBe@bVDPjy`4=O&k(MLIxU0RL&1Q8I(hOLNXoH1{@uUnZOyqGuXl4p zo01hGn{dFkER*b(mFPvkCce}$xI3faXNuGrVn39|LS%CYiVR4>%o1aHde>Q`lq2V6l^qh=cs+P#jR8#t0$IASY?B{M1c56<^h#%OG+GpCw*7&1mil2Q} zEHyqC;LK=vazt9qdT`>EeBVV!{KR^D)-+8&QEd;(rt(+GHt41T=dB$z*BNia}TINTl93nObKw zE(%RFnoD&zsX9$gbI=AiuTt3$@Xuc@cJf=L`cd_38Rb;axIe6r-o|`2Ijl?3~C1GSA zV$)u4r$I%Y)7?0@-oQ&C>09qifkV&rG?(0omoC7~B)ibLULC+@{b({{L8wPV;??}2c+ntEI5a(SODF=GFK+O6yk{PTakxp= zUFZY-UTja_a-NBTQ}3qPwV?b1Hqw0zdQ(|adGFb={R0Gk`xImwT-XAca=+_C<@PlJ z&AoGOMQzs4a#?i`tJU8`4!!v=62g4DLp15mbA z9-+J02Xj0>xcnTNSN4Yc5YwLV1*8Tb^Mkw3DR03*;xDz&@`+w2T?3@?t(H`;SXmU|uO(Z=EayDG!e~p1V(Pn- zXrJGZk9Wp$mRoWTJ^8TTm?Vj3dbVtflmME7u3@^OxJM`M$jEvX5c{4o`ab_ciqV>7 zZy@vX4AsrImnVd*P&-jS3ax$F+0xr((aWTXdvCJhs*fbLWAl6SDSj#Kw#F36tjMLJ z{I)M$L{>0JsrUvr@_^`jDqnB;9*w>g5|!b#Bb-d0$yLi|`xCa5ZJ13;(ifbEeDrIaJtdBK7wgI%3jfZ46;Hr@4`Wo%B}Up@KrGm2^VZ{4eek6(rAU@*!2D;Uy(r^Lxo%fDpd)*bc5 z_PmDOwFst=kwc{T$lW_`63%}MJ}8tlE4?Fk8_f|*Se4D2%q>Sqm_0JxpT1lgE-=PS zF(#c_Q;XxhIPwf^Z%7IlaN6MSFrV&`<2QQllg{Dg$rPqjXAT$E;M!UX`+A4Z<>c9A z%5Tmd9+3hx*Jg%h51*#knucEaxeP~>s`1V|PnKD1E=HE_6KrC=)_mP7b9mUqOngOp z-0ni!Tjr%imb1R8DTnuObzeZ+ZI(K4t%)@bQO94WV+nl zv%cKThIoI!%zGF2=#u1GQdb-|uIcSY`EjwLvsInLT4|sqMBN~9INA(-BEa9y`S*zQ z_)9P%(o5A-#ZCXyH_1-^`YYArub-)x}fH3g+^D$7B;z?9Dx7b+Ib&{Db%y zM6-GS-Q_9v?y#RllFH)*oj>kI>I40bFCy}HiRZ+SHaI^ngtZo4|8^JMCF3?P@+xe`;+`9^jp?E)L7TczpsssINiW=ob!r{$yME>Y9))q&<1&* zq!u?2M=Ph&^U7HLs8^ydsU%aApvfe@;ils&X69l{*XM#1!WZm{q9vTB->)q=I?K78=)yvh*+}x&d0ww!WUn=3HZqBmA%_WHQJMJ(!$i3{ zA{)6Q$gVT*$YhV0zh(ciIs>d9?MN=|bpoauzDNK^O8A6P9*0M5Z#WA*l;@eKTytOH z=A=y&jV6odP~O-V$ap4lBsHBmPCz8swHn6&lX9GnZ%&0>s$Q$vG5%|DoRS|H*vesj z+R)MbxCqR=4sU;bK0?o8Ewes8$d=0Vi4|O*Bce`Bm z-7Y>#U6pLlSO0^#w~C7E`SyLI5P}5=?(Q`1lHg8ohu{$0rIFylgG+FCcj-?J^QKFs9M!MX3eU(x@*?@eAjBkY_L{U2T=rUwSpkNy}zxceCOw-rv!QPmnADwfy~=@S=lT6T zNga3Jxgi8DkF)L!V?8%_1E1HL*alcLMLurEwer68<~xC21|Z?s+>>9Vr_T4VNv<*F zNW?cGURbKMu09ym%Vxb$&r1U}eu~`y2`?aZovYh)KwfE=_siKz!KlndFSCa65=!7*aNJMK(q@0-H7f@--z5}y#I=Q;3+{mRg&Sk(?Q&5RJ@_B??kq#1Y=8yZax}t zZBw)|pv!eEsWVRH$f=sGJ2{pR;9A+eP!UjN?O)^mZ6qOP{FasPg_V}jv_7ATgu-A; z+~+3;s#B8*m$qNzevV_hEul-0sw&Y;DRMd^B`@-a_Z`uzKgzX!q6cpfp2cx5D7gC> z^o&OwdFm(O-Jh}`kz zj^7~2f&)3~@7ul6T6$h5-BZ{bE5Hm0jyP6(qXia9Iu0)CCV7_kKsMQ(FSKyxT)vl; zgqGctPRH&o6h%x3?=|QN(cowF-E6E|ygwe5m~}2L-L|Vk7Sm}+Z%kfDyiWMM7K?sS z)h)AP4(r?AI5zg0PlA360b_Ly@M)Osni7BKnVI}Dz@t?5rfu!7k8y%95|MH;yW4)Y z#=Z58@t208#LwXQ#{?LUhwIRfEx-Z7qRM=e>!Yq~QJG+`AO5ue@(Ex1(Kj2Du1|#F z)Q*&Ewj$Y__dV(@V384AY!~IjO?q zbsPI)qptPB_%ptn+R!qHz?8dCHYLJmhv*WruZMAn+WIJZ5mkm`1O5!}DFo233l27V z{PI?A5Zi|PTwN8h03&wv#YIX6u`7NG?@rzDfz6=)^xsCs@dhZ?Lk~1wS*80W6hnbg zEiM?qAQ;mQ)!>a3jjrMLW9*+LraYa@E1hPh_z%3b>Fpox>USozzz~C>cSoMx(5uTP zxMn%m26r-rzyM^F>tTA=|CV%L5P5DXXQ-rJ9b^9_QAed3Hu2(guEm?>v-kOD$KA8 zqAyx8W}-HGI74{y&@$5=z|C4x`1?y5App?rSl`o z*NO-9-9k-dUXu_;VMIrm)R?WnIgx5ySaUccIN&dkXL*9|xn0`;cw|R|S%uyQ`>@f5 zhuM{q9KuG*&{y z`GBJP1H(@|M|h;;Z3e`89xyO&;Vbei0^LSP8daC#az70Fgn2->jexj}V-{Dt1UNEc zD)N8#H~w=@ZLiYn??=;PRto-!M;&}|v1ZHeD{Cxj^Wo{WZ^^_HLG!JVWRT6(fR95_ z%&Y-8s5V^eSKY+~L=^_H`z_+)mD?2^y=klj;yYOX)?L$ackG?nF1*k$y_Je-5!>(N zPq6LC)N?$zzb5)2mj!cOrg8VhuBg~9sgxH3nyEvq#29VYOSnj+Pr}Nr7&l01Cob^p zF1Qh)PPq0n_>$YQgM=*jYM3U*zh>ul0`iN1DPr34s?#spIPJ>Vnz`NQgC|cUU$^$( zG_O)-;`6FohhfX=rmJpnFajScD2-wU<-gUAUlZ@!wESZHkt}b7Ag2Rz zOyGaLwz}x`Vi{AL{QI@b_iwN;@AHZAcfY)SQ#%EHdE9++iK&kdrI)h{vp6i{@u&wI zp4bW=iK7v#w)5&L(PH{*6;D^}TGtPU_aykn7~9(~Uidl45&_(P;o;SUz1{(CMxybOYlRLe;7dJys=d^|O=AZ=EC1y2Z z++_N!7YgJ&xE2W5fUCRU-tncs9U^IbFl&7-rn)vCo(|K+4*?F|wXhzOK8B zA%k9>?_F7df>$uRK+2zeItQLd{>YHq)G)Cj&kt~f@Mti4_f!9G`m%5iA~RTmsKa&l zQ&g=s`QInWbP(Z4R)0W8r(U16nD1bA?rk<{BMzc&o_j;#NHE)$48A%-u!m4OM=vp4 zjbH7$6~JUC@o`ojA7Sz#LpA&WOX*_*P3aYCD#w$GB9ueTiM;p z0_hL;1&FIY+E~)^1AyykdKEclb-et*Dl=M^djZ?*y^}h%OvB4$b$cb}qe?ZalIzm# zQtHBqqkZ2zFp=FV8e+A^RmV#1OTeF=44Su>DGkHEiKjSQGOA;y=CSIw=~j(LH|le2 z%rZ$D{J3+BW4yLM?nGmB;8_>yUBWT~?)B4Un&g7NX6&qx&C0+$2RiG5%T>45QMX2* zYxK%VND9A@X2w-i({g78cp09+8zALf;}R!9e3Z$$m*tB%YxbuX8!VfE>{fiV3itK| zE$N!AttxN9EsuadYacB?Gp+2LXt#p_6UYeVGhPInz^zT&@S<%YS;uK(8K1YKD^pD8 z%e|j9*xnfK4-|}<7a64-3Hj|4~Ou=Qk z2j<3P7M{>Xeax@OnfsMeu@Z~Y~7Kk>B4bN2uPWCou42KG~)%?S>7^>ww+zyq_XP3IQHzG6jIG1 z`uVn}b)WethI7A0sH1TP95A&m_a*#~l8iSvTXkPJ`Bx_I# znOR#gxP=|*j2fi*boQMSYG9k~RCGs(V7cN#QROPD=(;z2$+IvWZnp=aox1Lzq*aa~ zr8F4yU4Mx?rrZJWlSJmXiu4oQlh#Tk03BXS7*Pj83;eG|B-DUc6+r6xtbljlu7ULS2oZx;WxUqH@olt)qnhKmc*ZBj|H#{Wx{PDJ_v^z*1``y% z6mqc$nZ5gXd|;3(cwJsip8a(+6Me?)Xz4UI5RUBQM`AzWHw}*K?9(S-z(cU26mStC zSSG2@Uq6sm=Z(I$F31M%eepX<3HQUj$L%I&xTWJ?oyOnmUpmw%i?cbluf;oanOh)_ ze;q%PDR>J(w?lPeAh&lnPZa38CO8RxyX_O9}ey)<9toiuI^ z=s&^bs&#}TA}minkD51mV>EUdQTsfx3s$>V|JAJlJ5HVx6NSV5`y4@8dG5!^9P6X( z>*Lm^{$>(Nu65U*f{pfY=FPG%^4`E}y%5u(QVFB}Yuk9G(ebU`xnqCsyY@Ek6U{H^ zaq(fJ&{c&;aMYU#tY;k^stb360FqiE-kWL9IGr1ukR0mHmC+d+< z8cKJ3ZujS7$qgX?PR6H-WfO$34+77G+@0lvmqYVMX4aQE*Y}UUfu@t!f2<&p90`4` z3V;xQMF$~p@6YBPL3c&N=UZh)*(3-AbIdI6)0^5j{2l|Ir0aL^4xQ`uzE{WWz8n+l zgKEs?pywQV{+~X~+Hkp8O}f>*kwnAJbIGR9Q=|9pgll}goaV)kAnlW7)4%=wplge! zf)U9Q`2gE!C^7W5Z-lR3Pmm`}#v&Gu>RJjGL#k^&c8s^tX>cdHGUQYWfag!HGb|(F zpx7Fv=)#y>XWw*x1sUSq`|F+%TdsMI9Dyy6RyvHg{pRgr+f!gobPgbA%iB-;Jat|Q zH3>2;T3PWbB8)>)Xlmw*AsSFuvgP!AOhjJ|PFjejuPyyFqytc$mC!5C zb9#RPQY#L)wR5LhArzFAJ(XTFxon=9pM6_da1jE;ip^aQps&oaCl!BS0<23_=KNVm zK&DYCa7YxsHz=j;d{vgy@ThU=o9ra!-S^vNDu5C}>Xq(U*dYuN-~~i=Cd6zLZ$y zH8EcGP_KE?ow5#x*yI{EgTh^sAJ@)o9E=|nR-5nLmjiZv0=D4#)mQP2vrpJ2moWcN zyspcI%{~5K4SYY>qO5;TZpGsbJEeh-3-b2 zg^L|O0B-vVhn|lrZ&%ka&T&i-&r&ZtU065R^H(rlMXIiW0`!%muqx1bYya_C7f0DK z`}}=}%^VVcU+9%YcGHJdp*|(P({yo|hvynARHZA(;H@C!?r)CF5wG!1Q;ff!!dLWp z`7Tezg94s~;7fknR>`(~Tl}k0+P%qdXML<d+ckDOS;>y}&1@I!mSu2O6M~bIZm#2lca_#{$8*^z|`L@9x;xQ?=M~*TuoR z+ekhHi6!mZY%uXXMDDzZo>y9{>#KTVKkjw2>BmBzNrYv_okPaW?l4V${Qe@UF^@4W z>*pK=yD=uP1j_yxHsn5$0}6gb=hrer1oPsluNAlyYEmqc4YHH8f7`+J?d@Oa=#N*q zDv+$qYP6xgae95@NM!N%fJKC2n-3cAKk>Hxb3ojdm>hE47};sAq}f^>69r~2Gwn$f zZJq59oYlM^m_%>lhGwTGht^gis$%vy4c>F2Jh->|^tWK4a5PC{V*N?+_V;n};Do{o zq$X?!hcO8kJ88TZyT9yr3hoe-6|NP?6uoL-;=pnFR}&-# zA*yLI6xgmqYOrIq^Ks~fn~(&Zd{7Cd>*oNl8>hc-*zDQ^X%N~?0opOv!H@A7)iCkq z_v-s-zl(vcKHrfsrDG+Zh+WZd-MKAzi&Ye-nm-;h*p#qYRmTZ_C)#%gHEtVB#JXAu zJsA$D7m-@MOst+$_&{8^Zhsu&Uz~pKc4efdAhhAPt``uM8wQms6tlQII#vZ3Yjjzx zSxt>w;g(;oT`Qdd&zNrbxNg!Q7=^%f@`0PRTV;-Qat5vdPs$jswQt>U$Hv0NV8wK$ zu6wBG)-P8TkbBWRUs0#$!*$06T!ih)??pAA=5*Gr3AsCl)^WW0r~divoDVIrA-H>1 z*7q9e4dZ9WcItdRK2Y=e!Fic1`D~HB)mG8JD#bZw*5mIGkAgO4ZI0mGN0D+%I7E!{ z`W_OhRqR?^^yDF3>_#PQ{8Yu*UDXM}Pjlqa0cKjc=N!lgO(5n1xGG|7?oC&b_hlB$ z4I^qg7Q6|0iZQ zaZ1nE;#eD(^(+Fv^~`JBB@rd4xW!>#k*0vUH(;-b*ve9 z)i4|Fq(62GJa@gmYZOUdzvh^)<)0&XfbyP=f2g|s+43s~d}dOEdn|`&P~AhQ z`KoLhT)+3h`wMxH$G+e}at1l%_W{LZbY-(+P4BP?=WxR#1tx01>saE%&8k^25pSaGZ&vr65N8NR$;U3?>6fv-SAO)$A|g(kg13_bRBJA zdi`s?*6^o+#^T}40XCReK_%OQFsacfp|%cAMogwkZ_;A)l^fGM5;Za!%Z0Ar{6j&&V_G z?X`5@U^(0HSa+(mKrix3^v`%d*D`!re;hj@yW@F0xgJ*JC_IaHJKl6C&!L~((PJ5- zFJEx!nF)2&j<=*}g$Ql#?`-TdRHv4j_uanUJ3{hZOrDDJO;4JPzIR~+e;2Se-MgT% zsa7tjDw$3NPSgP>YNqpqd?~6TQuXoR&2z&A0ghDx+8&s`pK?@W#v40jV(APl*JL_m z`mn|ttm>te`DhZWm$lw!F*1{4m@hARW3f%-tRW zE2|H7_1wlx?dkoR1^K$5Vt3_P3q2ZOv~LCA@4sPBBz+m7ZtzPLWi_kE`to%PjG-ipZ7hcu;Sk zW$aHitDES2d+>8~gp+i>GurY|SIPc1hOVT+vF*ZYCWs4kZ8pe{_4MEBTYbvO>P)-8U$PsQey3 zip21!=S_#)#WwK@h<$MQCUSb>z%DjTJUR|Vx|-hpaOIlMC&tzJtZ;jM7xEvQE z!YB50zE{%4Cr15K&-0#rOs}N@V+JM2hex;_~l+nTY8Q* zElT@u1{DvR4-ic4ZYYwC$v_~;NKY~QrOapKnh_R2WsUKH;qb)}(XO*|u+KF@#pi;n z(+AGi>X`H_@_@cXaD8y>>^{;`c1v8heWX34XLoZie9HAL=gWOepB0Vqh?T~?oTFci zpTO|SEr*4%)mjWdV_Y8OmvA%L`s5&2yUe-f)1_>&n*3pJi>cOf9TqGH;P>WwM zF^_ZJ+pIpOvD9oB^rf#Uk_gYZb6Uc1)gvpbD8?4%(`t8+DcG?Z`yv(M`K|ITEarZ6+%jJ) za$!!(cR&edn>==Z@-(>jpdSu#4r=IKf}GTm<*^Q8ck{_l1$e$T#MFlfVsbW*Qpkbt z9V{l}Cifs8?`DPj6rV`{(px)@%x50kX^ZibV?U-hTUm>#-^aro*Gt2*sswZV@-Dan z)aC`m4s7Yh-7akduDGVg3c<%0!Au827EAq{#bOE7!=zEhzjlQSpq^Orwaf|^j9@dz z`3rUg@ejut-1Yjg#oVDR&4mx(Ytrq+Ug6Ar-=K#-@P);)?K#1v9H~d<+R$|YJ`Q7) z=VbDN{lfF1!TZ6&b-EivjR=Avhu;}3f{#R|L1&MKN1ZyO_Xg0&Mylj~L-KP~u&++;-DH7asPj!|2 zVO#zRNUq>M2@a>N1iOPq?^li4p8uPdJght@w+}pElqPkKcC91db*?v6&+SgpBUm3W zN5%ys&z~~N@>bOMdL(O456Hj)ach3z7eP z=I1gT*U9PE-NHZei|TY;(1!XuS6^icre5d-VcsFZuVb(H9yYm~mb?Rjr0$K` zeEUYp!8KFpPojZW!}tpqitwr@JT>w;1%lOsy_N86{~^&YmVOZ5ig8?+#$tKP6`gDR z-}DOU-THpo#7DxLueq+It{+s1b!`pLx!?nzigL6^-R35%K_?37(K1$MGAwT0&z=eH zbk)tc?ZQR~WeEe6hDoj&9{USi&_}`=Hr@b)36;b01no4D;fgi^ve_u4z7LQ0=s>a@VYIq`xE5%!Ee>T=~rB@8(ycrseaG$UKrO4P)li|JAn}vT=ob0 zwdHHgM(u{lpUVx{hVviYKg>aVZ}5@G(b-T%NguS&V>a~v@cdB;GDZSpo?$NLePLjo zVVdEXp`8(#VV>c87Ukj8AQKekrgbwo=qzMTs8-NRa|2Ep%6ZbSrb53=naPdQA(#)F z3R@D^RNypr;fXyqn_8Pjn|Yh@o0sPtn>L%eo6OzM*d)w#5)iAeCv5Hf^=5caTzG1@ z@p~|d5w1UdC*^~)ml^ChX^k;)dQGuHu|mEg_6=1JH8u5ad;v8MwK}!7RCUA(fi96Q zo;Pnj=cWA4=s?Cmf4n^*BcTnU64!gqFe{Q@9Q0-?4QGp1mg8fCql_bt8RVm4oB=M6 zZ|_jw6%zou=7Kd|F z8N7kKe|Ycgks8rkPA5FX7cc9zhntfGPJgiy))o2=7Si?u1S|&I6!uu!3YWA=OWDfL zj8PkT&KF5()APrVc8YOpW-16O{V4 z25s$QCl8DOerez&<>Y>D2;Gg^nE#U_t~Ed|k8TmN6e1Yj9_o`1KuA!FKlhS57#@^M zr8i7(q&Q3|6q-y0CXy-oaNF2SUCA*gYSDHDgGy_~GO=SrmP77C!^36r0Q4a$7{;+6 z16$~>GIaOxTw+tETP5r%0@N7eXw(%`g$5c7e9=U+{=*sB)=y<|u&ND}?BoR@dR_xBl9R+_af4RrWJM}97 z)6bxL)FSqZ+EOR;O)nMWSU{#W%wV&;I}R1hvZC4AA{%LmJIbb~;}$U4ZpuieAza?t z2^-i7?H!IZo`Fvp0DMx8QKF=VI0z~@HQYs9!6tph#dla= z;SAtiG{c0RM3>quZQPd!z)>hkW@9omf~(tD5MMy0mZXr;;KQV@b)A9a$Elbz?FfYGsx4(RZ9>cI4-0j zG@NBIraq=-^>cdo@MuSCXS${ssML~iGdo%lRe^dSq3rJAuH`;`BZ|LU>x*3i zEHrmPLtV+q$v`Gg&9llC)wZh3)S<|ss$_j8KJ71^261RF6!s-|@{T z_)UNutoJDj5e4rp*U@p7=+(r%VrHvJ({`5YU5OOH#n(2_aQ@tD%sf_;X6WqDyNKJs zG*-@Ew)HpC23JNKT4k+0Sx%)l*_oe9RYYYFuYpx5yVCA{tfK#%Bsra{X$Cc47gdcr z_^%W*CfR&^y`>+ z-eC>g_U)?X*R9=J*In0W&C|osICFsi!~U_q&wh!f80+1|7Ww#t^Guvb07JGePlLI@Stv_8`POf3-#cg5e&n7TlGqt{HMh$;3}k?;b!H3nxWX;KiCuy~ z&6{6~j^^NOg#67=rS>X<=@IYQv=0 zNE|^~KZKChBJ*>+O)z|@if?0kZd@Ydj=gqp7v#>i(v#zO5I=JXf2 zZ#DoE+n9@hZiE0?-SFY*Q*j53T*C6j_E7-I;%gr)%ab5K69N*eE}cRvF!9<3O~#-s!l?0LW9pzrL$mermbU@=9VaShD(*8{xl3qY@Mfg_*BbV!sXJ9k(MEV9jTVVD8U_+rB$_nH0 zRcXb%q|Y@B45Xs$V>R!%!p~Qt^gVy!_nyNtkNf$SzdE6MZLY-P^x~u_IdO$&A4<-C z8_<{$iu1irr79DW%zD{JF*8s5TPI{jmn$U&(*W#7iLT)~$AsohTV#W?Nm z9Tz6bfey?v_01d*#+w@AA=}43(n{MXvv!>Doh?MyaTEDn0gg!@;DO!5jq2fD?HKk% z)OliUX6+fW>j(b`c0nxvMcS8!@4&C3&AJJ8gxFAIsODd7r?S7it5bO8LHEFDM3hI*oH(Fd2d;N!B{__yUwTDP=b~b3Z9-+Yk?07uN1h$y#89;3-k6 z5GS0TBn*JGiNmeSATp(g$Yw zV^N~gwU`kH9wCp?1TB(%g59lB%~*ZKk_)Ju6JW_cO6pNsEaLdj`&=2=mMokTCDE_C zGaEP;hB*nsDYg^-5SUNIQMAVgR$GivM%xd5&E#4z4yxR=+;I$Jdt;-|yPfD)YzwZR z>_1toRvI$Nbe$O8(4x8CsfC2p#-vFOmY`F*iRSR`G}MH)ro^4?72s3e6#MpZ2h`hg zT@r3TpuQuIGY5>$H}9BE2E12!kB4JvfNHJ((lsl@K|SfCnm_iof$UxAU9}-Y;b`c| zTe$-_Ns*MaKFDD8qtNA#SZ`ipg z&6VoF0zH>Q=mBsuJfto;w4G&-lF*RpuRp#6edY1W+i0sxw;iKWYTx=}t&es6blWN) zIkKg!a4fSva#Tw}Mba-L4_vX1`Ml{9o>QV?gGs6(q90G4?0@=8mZl*n6@ih9>rr)F zCO7M*ORAOgw60ns1Ro3LO^%o>PonB>O^%q0AGV6BSsA2NqH;XvT{%1&(nLchd$$}7 zx=-SI*V{r`TH{!xnv~4f3My-?v2Ag?qVt1Qlc+-dWv;#>*h59QBeNn+anvWf817}0 z+T^78^0y7u+QfXK6_^l_Wcvw`DjzH!3KqIF+(d?aWpo|O4!od9sWXm6*N(C`ZP>o)8qkuEJ0=r|NySV9(ap~8aFXqC#yGNy$)a*< zF<3!&xCkm{SYpDbuqZZ7SfDoN#K=oeN^C4|tdQQ~)>r&dNKa1!_7;Jnk*jpr*rGH+ zOPW)p1X2(@E**W53T=_n>7Gitlk1;ru@GJIpio?2o+@r$xOa1Fif&^rICHa8CEQKv zid+@f9wxZrgfeuCSRBX@-Y^D=YqU-`p5Yk~3 zb?OSSo~JPEhf7*mAy%IWN;_!Bp!;AtL4XY_>lPc|etLA-o7>a}GN9 z27~$qz_DeA`L)3LD6|r^w!Rcv>3_D~KVyD<3#!7<9kC#F)hHPzIq;K^osS}Q`esNM}kE$z6pbL2(HJn;&}WTo{GS#=lLp8Z#zPc+svDbDKH!_ z^V&m@?|Ucy`MS$+JSG8+C&{JB*PId3xCi77%ssL|gZD7Uw)W^$v;-j4rNvjjU1b>% zZG z_v6F_h?}H~^;JGG-fah*Wi`Y80$$4flthu2P2ledqm@#lMGwY_N3)?)qE*kx8(XxS zZ%gRkGx&|9_jhm7Ab;xBwiZ>AVc&w}#whb)GL6>f=VQxZP%fJ0Wl z!WwAeBj+ciAqqkk3G-64#7$0d=^+zNdFf#S#?(4a<-Xxfmk<(0oE-)Mz1a8&PPy3R zK~AaSA!nD)Oo|1jHu>rl+udgpA>LruWv(y<<{-AW4q1G_!1iUDFr~_ndc8)t&EIfO&`3TUU#2oo0VPj#ENe0i?_J6Fkh4PDn<-KYYkouuIg&Ov|ue9IZeI zC>b#B0f*Rj-=8+9aqgm}o2ewp>`JvM77rV%JS4I%=5*>|I(`sG}@e+ND>SttvWZOx;@y3eVJU4{m2uEOu$j%M?+I=>2N4;@%UY}2!~RK872!JNlsQz*b!iltz8R1;N7<5W$#iwt9J zSjI5KV<5U|tZDdWOXW`$$OK5pt%<&(ije9tN(xaRz1aH9M(1%(MM?Ds8;~GLGqVy! zgCjTON~B?@v)6mF37>mKOJVV|uZy4+GVQ_l-r<9fkNMsUC81c=~^oDzqy)zNxTj;V^2+ zSMr!f+&g|Bx#~`NV-bu$7eL?Djk-KFMYIv>hh5Kh)9pXlq~){u{Y{m<5x=S(tr5T6 zQaPg$KTFT4A|R8B^+cxADJYL3ayJZZ_{f*I3eQMn; zQ^+(%Y!4UjF%5s;5FM8I_q5g1#jMrE!$DyWdvdB)Ns`tJ;tw)O5@)g`oA^%OQBz5< zuwO;IHlA%HC!oQ|opcK@R5wJ-&d+uoJB5m7KZ&19HCM!LSg~*C)tVaQ)54cbU=x@a zsfT!ot=jxy+U;`^L&oP5rr9a->q5TqdkSEI2a;8AE{Rd$IGgX$Ic>Y@N4e{Z2@|5UT(NyKw_KPrfZvWz6*FLVH%PEb=Qx38~%@!Opw|@BIh(ff(vhWd5 z_bc@qexrt9zRw2abY7l{*e}+?nlNpZanKaS2Iy%tsD5sU<^@jJ1}Tyv_S(2(gYk^# ztdohn;?#?qW9plgkfix4+`A5cTrLq*R`F^*XVT0h1+Qt_0edrt`8Mfizd3th3w^_U zpXx(N7d~Ld*wD#Q1{;CQieDBet(G>=@j>e!`7AuF*os z&2Den;!;UX)E>R5_`~;-jM4>@OI^peB?I!eiqbBn_t3H`#YY_w*=gebVOR5qW)zMP zXBSJ=-Nv>=H_^JL1Qvq~_gU4C>_0<92Z^mqF~~aOIH-`?FxHYY6zXi7NjzN_#|>L^ zKTX4Oqlb*(NR3`f@5sC9yQ`bwFu=s!`>CS{IV(o@~643?p9)e4T# zZI*>$e#qt`dhKaZVyUKCx%m*0^9ZS2U4f6+UtB&W+oxDR&F%^%J?P?u^a@ z?{nO1ks*5L^3xfbiGhO&A;B3xq3^eemx-5ryZyU0LvhZNXL8v%SlLchc&O63_`k7- zcnR}*hNy{ZNE!cy($u|>bd}@5iTa!`T!EV8@?LF7J%Z#< zl(l}EjwAsJxk?D5JUiYPnPr1J(=Ja+6C76lN3COvk@F6-=;mow_OO7fMN^xfRN#?} zvmrqWja-z>Op;tMtdggZh1KPzDE!BXoL09AYrK|Fd9}lqAF7v1(?cM=}eI_1@y6HH@Ds9s0evf;S<~Lq7uf^;+GSU{OSl#enUfo3*|#?kAZAUem{?3 zXb;{tADb)+(`@sGQ^NCxKPc_(XZ({Dp-I1`@$Lr!(VHJ%QXemW^&y4Za^o#dM3RD$O>V@t+9wPH7-wfBbcHCFA9+&$toNbG^-UYOL@i0pfwm_kkjomq`1cl$e~VuKPdN9#_;}*rVHe=}FAu2XuHzxLwpcn?fU(rtrvO~hWp*?T z&)MqrX}bed>vkd}pWF%vxpdsgkJ_T2AVoZysAM1eyASmkafRpoQUQS zvE$RdBP^AO>ZvE-Ar6da5qlJS*j8ay?0CG)!SjZ_4tP8UBRHNe$)BzqpP-KO&*jqu z!zR;p9NEzWY77)2%>7tYBya=PH>XS*c;Y|0euQ>QUIDl#10MU=9}gjtdXKPv@Du9_ zDk1lm($lzrhZ-=#cpmra3DGb1xPJQ-R`F<{O(ry&{6?wOUp&gYWKeUeF^9?KkPe3b>^DlqZKlGq88nPyd2w58B5^qyM}J(}-YB7qb^Ud! zWoc=f#EZ?0b>5jGw$IB|eQ?XD2=gD&d^6hWzm*)^5nAw+WYEB;-MOWLYU6br8Q9!5 za}bL&q!L}`Gfyv_p)fVjhCQk|?9?yU$R>A{^H!-^Z2M?(IY;NmzDt(pHYCCFVdsir zX37@M*k$gi%lZ(#kRha8P~(;qyvtDSdZHPV_*XL~SSQxm3!Bm~_u@`2aE=xhub*~c zOJ0TxVOEN${zkm-{%9pP9q{dAPAbd|+wU~^tU*XSudVNnkvSW6^sVX%39$U4#JH*b zUj2||=?*1@!CLQ5Zu8+y6f8*{{DZEQis||x@z;IwRHVNT+Te1wZ)nb!aljY(Qs2p& zcSM8A7~J?5C#)jVv+r259=PQsmL+g}j9ruuOQy`%;`c}nDH+aq_g8-GxwHu2Z?$=< zz|2>&6D6xUrxXtbwb?|~1V?hH1X5O@(vaVG@#SsO2mvu-@4ek2u_DP^E;j!0_|@?= zGB7#z+NzL>-lPiXVZKUI&l~lT-)*mul@?<3G1FU2jQCFJBQ|n+HmYZ}_&rz9I2Wl8 z^VUr0JLCVy1bn$_(!>Ee_1Tjvv%1i+0O@~mkdPqc?2>;wMzvV3Nh0* zry}jI$}#kSBcuAj9^8sieJ$!Mnj&?0<*$pHCw8>#n>(Y79XoP&+p0qLqQ5e*pA72T zL5Dpu3suM1vMOgZ4zbRKBG_LeW6Vex7Z-#xnx$JnA9>@7NlmU2kL+dKj+sIfT~Ubc z%6@9Xr;Dpht`u)3dF;`DSta}|yY(2MJtb-fm6e>uPMgTgY-G=WC!K5n;srwkrjS-e zLOA`3t`v*9V_eBs4S~^xX?qoI+Y{-$f8vCr-X*qNZ=6!vPY;0!Ar?g_$^MorpkwpdJ?Dw!0Q9VaK`&!#f-IgTM1v6Sl3det@PtMDLGz zM(sjhCVb~@QF+>*ht$%lTyM|yg>Al2opgJ!=z|b;7AQpa5tAB-8DLVccnfpio;-bX z{gG_81|xJWni(y%xcfb>nzDlnE$ZiqQr1G^(miZEg+p@@c_NFdNdtk7b|RL#ikt+Q zvZ>no{E3*tC1ox62ks&Byb5P@O3gxnSIi>rw~@2&P#b*aOQvSu(MRysTNnEdWdUB7 zqf>eoas;W9gIG#&J<2PDXQgk3+=pJ>z0pWJXlj$ETvjP-C0C}6v0O|o)}ecwZP#?^ z>}l7r4_CW0>9_G{1A|n2cC{P1iVP z@z<8rw!ajrqt^2J=6-aEvB$pBr@k>@q{y8%sQZlcR|UNUrW!_Y$WVY3z$WO4jk=R)vEH0xyhAQxS`0{!#I#FCM^+EuzJveal;d zkuqL3^pm?$C`dZy_1m;<0bVi3lL65xMLorY?SU6VI70P~m$syIDagvtbJ~zNAlYbA zc8uH8;TG-etKOuz`SL#UW>gP=_RoJ zv)p;smB5B<*Y<5EFOf~G`J0CF`<*1TL|T~s57apqD4R){n?aCtx*L03{fAWYK&DuL z0LwIb#}dC|xq)D`nKv70lZFDY!1@I@PTvMRaldk;Ga-z@uab9Sre9b-a%k zXpL!n7^}55Rf_=5i0K>#Q4cxscDu@1V;@jYAja@62J%u%o0h3ns~SF?2JzYSlb#H5 zKhrBI5`~PtWlncrMiA%_hjo?306`yz?LmZV^77}kaS;YZii7`n{W?F z*|spMP5md&tm3$~YA=z!cmeHBva5m2sI}Z2JPEgcO7tR?20A@s&o}3cZObEHnr-G5 zhJCsP05OedW^FwR7{j#F`m=G)`R`E!b6lbeDtR`t7i3#Xf;fhM@>``YqkneoJLG#a z`pC5P!Vbd$3;1hVNR`y(hw8^nqAYQw>l1VvrKKsLHyCLk6yOPJZ8YK|<< zPs<1RN-W5#?5Rgk9#k0YK2t1Zb@keArV(xQO8%2Zr1Oq<{G)NBB4mlOp=eEUrsT>PCsC&**k`xP%kfvuu5xGW@Ay%%BSdI^ z6=8g)uCsebP`vle)~>;zSEOp_!ux2(9Ut8iB*Et=pT-u8ZZS*!< z;Qk#F)ccv;EE^Lljg?}}oQeylYnBG{ZGvPr^_wq5=<|`CS|q#d0RuySyCCcQitJ$0 z=TQ5>5}X6%CZ6%_C3BV45EnPgnZ47n$Ek!d-A^`AR37%pXsfH$KFRzDahfCZf6U^h z%0IonlSqh{YIRtDOgGA*E~+qae96b<3E{Z1u1dZCN|8QFYSJIo%EwD7!3vFaV=3j} zYdR*xY&G!!yvb)=pM>7F(%IJq*1nJL=Hh7 z2-I>WgISbg_+!Z8#ovYPNFpbjN9-~gicmAH97lL*qQ4sscPj8FIVONd03wZv;Anc& z>p9DFEO|jmb{nqo>v(7vp}KP~;>**pMdb*tLFhwZSqqUyfn!x1Fbh5>&hV5Me4r&| zcqgItEWPRw714f+%Nuvz91!}9mC=4_tj^sedC2>jD&hEtEbSM7v^IJuVs(CrA#S=V zj=G>Cpmy^YZUr|el{2_$io-}o&MV+JN|DDI3kKe5N>}{@jvbCGv+!P1l0Q3sdD)<4 zRO;H$Wsz2Ti%_SWZjT8<0gjeHv<$_X@5b@kn(0oKtmyb^R%i>mNwaSA23^4~mWZsV zl|}X{bjG4LVpO_LOOix69QE+%HkaU0d#yNjb%=6FLv4(yR$Gq&IuuXGiuL=Ado7)W zOW140*q`yJxc&4go@F&T8K~PL4491Kj6)~?E#<4x3K&L@MVdTYD;NqAW2tMmW_IgA zcA^!z{viC*E8|7Q2mC`o%5Z3K7RBQ%y*A7za+#%`9%y2ljrV!_;{Mw z!;{z#37A&=yg#cN(c&;mg}Ggv=t}PChvV`<=z5rJO@iCrp@vB4;qw*an6l|O+|DNH zZ2fO)jaw8YT#+(b1VlxP#+cHHBgE{ZMbT>Yqf|`xKKHUfqMFruJ2B>F^M)QOUe5{o za-kBn_0yFD&vxffjH9dOrzddZMopv!5_D881nVU!@}hp<-zWtyN z@>vfl0+752%LYJyi|xB@XOrucO#P7{z9s0!M-p#-!;2?U2M=J$%#U4?y}~#kk|H)) zT;IdfDI)aD={QHs6XirgA-TfxE;(>|V>ga0vTP4I9>&l3;t%ANhwXN*cJoM~T*d7( zA}roXRoKQn^L?LhVyvAg_2!b+oa)bKZ%Urz@uSx3`;(%~*b%7< z#>P}dFi-PuGCPe7^mr|N${%G6xr+?Fs@f;cxZIeAKi6+Xth^=KhTUcjmIltw zCWIl0%BRYyzW9FxEu2(H3seHbOJX`{ZR0{3L+ns)7hfP71Vj+`-&HVSjt_3&SjVZx zQ9;`0rj6@diTw=e#~=miK2=i&n_vQ|X!( zQ(}o_L_~!RM`n}ry}zp*XcO>v{h`c4)GV%J{Syc$O~`3gSZSBMi;A=d-UH{PNdu%E zt{UDtyem4xt4+GRcX;h{`VIC85c?LVlT$a{|E|kwFeiY-+`N`Y3+lKY;c;vMY$s6~ z5AY#TZ2`8Zgk~f=QsMTqrHiby6VWn$Q6FfU#zz=WelzFLnX3p)pykwcFwSc2)Qwrw zCrT5lFYonVg-l^du3WWGeH`R1GG2bMtB?z(=H3n@PWmc$~wVv|t7ylT8VM_?y zaRa00Z)e^Y{(9cive;oGl6C;S%OH;mL$>N^`R^ z=u)}zy8qZyhMcDA%$h=?w}l0R4Mj_evOOcx2p=#E3*-GaLqE9&Cd<6hE5CSd$l0E833zw!o>Y z1s>SqOf|@a{W9xr?bmE|zGCRsUp6kTGzyFGEE$mKd?|^<70O7;horl=??)Pwv=KW} zMRh_mT$$5^2+k5$bXv2TFwTrwE{Jyhx0?V-GY-ZdM!A?Cj@ZOc>A}Z(SNro!QUM1mqqwW)#^#^R4Vi~(#;Ec2 zj_6merfFri=`d?SV1Hh>U20yBP+!!2sQC=A|Jj#`toMkclSI~e$r)Mt2HQiEIZ$A@ zv=ENGqCI$+6bJl-nA74wD3@*q#BMw$kSH?#(I|g5w~_LPZYdMFM0tkY`g~%Iw^Ohp zRHunB!p$PzdDdp9#%r4=*&*%o9p2r5*xyIO=&)bc>}MoG>DeDcJTShBmQD{A>>RNW zgOKWSf{g`jT5t4sbRdK^^Po56ZEFIY zD!!jGokTy`BC$H5rBD-}vczmN`tPxif8A3OKb_+JReHMjwquvyS5QrcI2FM#y)-WHi`CoRmH|xe8j`1xx!cN%hx_^<+vPA{nV~tT{^&{E^Cq;^JsI z1EQ73-bt;U$4ejia?}n#R&@Zx#Sgcbibs7p+vgp>A~I1i)H!5bafbTq&XBLleowD9 za56t2LGrTD)43FS)fU$8e}EQUjm|Wc2(}sGuaHd+A76be-e2ClZ=G9xs#u4g?#cf4 z$)%a23~Uw?80gGpHoPhXDOF|jc3<2j;&cR6PJELkkGmrACJqsk!`wb7EjJ2s%7jSv zwG*&dXH(EuehWUd-KGtF)KKs?%a&S7TPFxH(^=O8^PzJkraFZh=4!D0Pq*y8qVdVrN#<=_~w_4 zQrVlG=9iAH8b7yUAvP@WPHjSN<^+2;LuOOmA?hfJ+GdYm&C*vib|W&Zzx^ygBBC>k z)#1~pR33$0CK!-hmm+U;NkXo%rT-dc=%~`k6sgKQxK65ZMv)0CqS>9_?Z6`H;fNtY z(h(Em^-V$%v%7}(>#{!TmBEZHz-h;Ur2ZmFom%htQ=ZF|6Y3HdhQNE_@4?gvy*{n7 z#(~{tZ)UXbD?K?*isl6#Es=5Ms?)_vr2zGVF4B<5rmq(H$-a#Dgc8uu={W)@JuZbDyLZ!)(|408Rs+U`+*97zpCAj z{b4P;Qs;Y6{NzbSW#^qss0sq_`EzFXcSXz9dDdAFvjBtdr}apC*T8f_fYQHvWppdP ztnQPh6|>2}G`-LCOd$~g5b*U@FLTI#oZ3}#tq@1~4oiqxGw}$-2zGI@$(qO2%G-|X zBWU{U&}P)Q3cNVyY)Q=k#tWf6D#-Y_K?E=E6P#Z^5;i<*1`+t10%IMc$l z@{@b}#`D?7D$AqeRp?GJW-(^loFk>-`@qTNXlV_JgbC8Fs>*D;Ayu0?HQDp^kH-|v zp@B^VUQ7hx)7R8t2k!pI`6PJI>_{$i$O#&pwPwuVTGv>Lbfp(oRgIPCSq>&O1;996 zfVNvFe|jlXQH^A76QNNe61=TrQBSWP9rQS!e0*&5(zoO)LG_kjdye=kA6@c|!^>hMO|vChJHV$;K7L zIB42xb3L>42bV3~bO_b#zvYQO!6xss+Hx?$Sb~Ab_*-PdFA>fO-y?Yq1-) zm^Nl}vRXX|JKBrWH6nCoweal%w$(&pbq zK>Ib!fNW867@Y8m#!!?}sU64DBV8F~L0K2|BT1A!d*RGTu(r;G=$l*tZ#R%5suIn+ zBEddAn`A(muu&d%w<}qGB#WUZBw=-Ccvq`r{mw$f;$asnkN;2c7NzQ7vSf6CM~ClB zl_9}tP>Y(0klF4>1O62nFJPnb9+mRt>J)r^5`TOyR_wB$FkpD}umUXwjEi!1?` zP>o58hTMxP#`z^Zp^dkO9$&b|AL680~JH8hJyGk4ATG7h=8yXHV;YTyF zkkSZHREb@usw@4&6Sr-r0pvE7>+e|ps-KA|8Q&=R`j}?V-FM$j94X4`#mV}mL=zwG zsxQ`6NPV8B`?J<0yL~OU2OuD9g|gmZ``xYq%a@j%jC#s-CW}CfpH$?#+#NiWN+7b8 z4L-_YRFx;k@z9Wrz)BaO?9P-7xL_5;zih5j-Qpr6<3Zwnnp_t-Pc1f96EPF2N5s^% zWhrAi5tL?9%d{kOt==V3+jDdM+<_v9)$tBP22w@YEKuA;Y-%Cof>hjsg+*ckZ16%R z98pP8byXQ$byJdb$m*Q?AFTn?e?xYR)H}TNVLqqj5bUF@psFmH<1FrfDzyTTMFk_O z*zf&F+0IdDux?WO4da3Hk*0gDrg5ZLDCwGeU#UrO|qDwWy(nQ|<*d|hsw8|LtDiw9Vdj@$ayrDnoE0DE<1sNIlI(M^tN z`A)**c|TFh_?*HrbLqnIX(v+~5J5I3Vo9s?wG%5TCQ-`)TA!=O)$hsi& z9f!(g>TqQCs#*;+vI$xbXa1N8I2-ZL<)*41RGbz2i%WQ6ID(shh{HU1PNK$AmR{X6 zE;;hziuSwCr?G8|1{wy{MDJtbsM=!hC$^YYh{5U00k!Sg86x8t`C|kE|io0k53!mE;AhthF zNh*Q@cG#~{we3jx#la@125=&lAVjm6qg{N64U-M9KfiImYFn%7Bi#Fh=r6` zQ4O5=Ya3(0u`@B%u}2>;yB(P`Ibf)Sgg@Rja}&{-1N4x%X_^!>ohg2khiqlSo1xnM zNa=BOi^TIFVZ$<|2uKEQc4EL+b?M@rd`1Jd1BS=*j@b?5i1|vzG<6?;t<(sn+3Usu zOPf%VkEXU$HfqLjM{0K0QOa z@<;?ah>t!R(=y=IuP%O@*2uW4Btv!6|27_2a0T~kT>LNt4t`ROADcc0GL6KErMZoA zDf>=9K$EC+*?!Gc)@k}MM*sJ|sWaxZ&x>mEq^Z|s^p?Sf!K+QbqemPn1Y!*`sviknz_$yKZh>KCN8rlQXQ)u|m=9`WfzP|& zivOoSm7-glTS3Cad^Mw9Aa&(!+67}@k5ut`&9}H}U4P)d#k^{kD>!ypX1I+190(r9 zEr6bHOXT)fX>ZqHZYuN>zVBYZnP2Sc@g*>5Atk#NeAN@{7w6vISf1N(K>2l{(sVNO zgTO(Hi4#aXRahd+vv^%Y_n6t7iOLXC*xQzQ z62VHb-7)3F5tUXZ_<@huML*j&lP*Td1J;F8D3le62t*jd(%#gFVY8&EF|wD+x~+rU zmG#@O-{^90_~Iy=uUKK%NOirQ1-(pU7l$z0QUPa6suHQOC~G2SJ^-eNxR394RPOZ} zOFFTP{2oB#N&tkPKhs4?07@@>&Ke}2J1a`EW0bK-NAZ`%PsZ(D1 zGfIjEZk6wiCM81rl>NBMNtMvj)Li^Vl~9N9X%gz{%iI!49DnU&nOCZ!zLcjI*<7M@ z*qK?+V=`s-3eB1?m2%}#`=M=kVH?NAGIMl&>b>Gp<~#AvneAd`{OJ9a)`EKeEZg+p z${-R6xZYuUJX11SaEIKSnw!$*Bw3LlxUhSSZcP=;kny%E*6$5>G*}iEx%^{*^0{xB z<%=dZeAt`gF1Z4=@mVCMSqy%@LANAVOmIBwl`r5V5>*3XM#9ppNEjhQPzf72tWbAI z$*~FY&12$Dg1BfP{G67I5tFqf$#h*-6VUKDJ^7v483jciX9^xQL0p2%ehQUll3qP& zoxYX~_7`{UKHk3by1V(aOCD9N_3UBb!x2{U|&IKBQ?968#FKt zS$t+^7ka>Ss1a2t$p3gwoRLa4))s;dnqTz@Jz{Xkm^fm<1{zkCty<=!zQL>s_3E_# z4h=72cegb&+L)!O=@~aO7-P)PZ;{5g=7OGicVhF$lFIE3G{usBvm=6+ugzCH%rn_# zT3qQ#CChmK<->EpsNOx-iY;yi(Tj6U<+(JXw{;2s_~Qusg8@xQXNP8Q2-rb{#jptC zkA^m0u-RteCQZR&Nm3C5b|1uDvkrFY_kuaK*RZfXyAw8xUPYKpGC;}40Avdzo#7rt z1}NhLI-0q!7y9mWGj2aczwzMji+YPM(IJbYG$Gj4S8u{sHZhz4^yg@$uG3+ z_vU(o<+;rH|EM9!=-VGW`8B<=FO|7QK0UtbkLsy*ykGdwXK$VIc}>^hzqN}F5w3bH zt-GpNW#7J+qFzV23jW}rm@!T(-@CobGn6hPPGiSU$Z#YmNd92MBAKrhtZ*rncdd3E z9;c4IJ+?gJyp98}8)Q0y%bPKFI0@6^rhz1fWQ~2k2m}M-TeZe*0apTIry|TImJ>l* zCo`%|6*3TN+p#f1gpfY1)V7J*y%pyR@bym%-SWhxS6 zE$(Nt!g5c{IA}S(_UdB(9QqA%mCCr6JuG7M+&U?seg*areMR&x-IK^vf8LP5wGM$@ z9kFq`0z>BtO^x~ll_4chizav9|aZk)hfw#GRxk(Coa<(l_x^h&akoA&jX zSmTR8qOJC9Nj#7JW13uA>k^EW4dCTTyrY@B=6gMO1rU4l@N+F^dk)jO?f-6hEf^P^ z@xUCR!}hLFU+BpC`OiY*=PJt+)!2XaENK`I6>mPeV-XG&?7F8sqSCNTAnPNQIUoxA zZNUDb{3QiW@g*KukvZDei}_?}7#U0TZGXDGuljG|-LmGX&4A(oidxgh03lq&mBX{z z=m>L;Ac1JH~p13iFWmhk&{jsGjWnih}(K-eg9~ZTbQ{n zJ8PHSodFDa?l-0IZ1^+RLSP>3hN9Tqr~YV77w^rgR_G+IkRV%GlYKO#)Ez`I83A)S zqgGl$Q4uq0934!(z~-DO&BD!wh*QX9dK!b-Qc+)936IBoYT1oy>$IRE&P)a?*|Ro6 zo<+ZBhQNt6^k~q(_6Zs#6Xv@TTRlt4gfd{ywO_2dG((7BT@wVQ+A}^;4{VWs{6RVL z$0jMQGcg;&LFVBSK%VC_Xr& zfhZn=7pyf_wsVHA0cqBw*v-%Nn}U*wEzg-qAP4%Lmcu%)wkFIvD5}amxBmWQ2=)wD z*WUx40nLd@ou$+qW>CD^s(REbhZ=N~D2x{hQR)93jU2jT#hwZHE0rQA-eEc>_va8&EU((M7tSb@-aWg4T3J~U+6B^ynq<|hE7`@K*(NOt zRcscRjm3}hJTf6+9Ak;Dhgko;gBkWsn1Ns6q1I|>F0um*` zNBiP)Z`n0Ry(=_+{Og)DZv}`otQ@@G92H5-NNzOuw|I%iBtB$R_Dl`u(@*-B6D#o~A`8@~%@gNE$u+bb96OJ+T<-Xj!%Q__ zX>7T7g`4d*f7SXMsW-W2A?fctv8ieDz-wfcV9KX*$Dprf%r~0Rlr6_UvYqA7*34BH z{C|H7ai{W{TW-_@KDgM4(wULat~|cjd3*Q|?K~~Q-y_=ilt`I+L<|ir?9nqMT}&5C z;OH^kWpK{$kQ!=K2b%KEik6qPY|DT>uJ&-@%w1k6QS-t!HzWkHd6;FL*fZ}h!sTS^ z*ztFMMV_g^p9q`Tx|o9t!$RzXMdq-1Mmq)(_{vJ00N=3hFr@$JX=dqA61 z&Vq<~VwE@#LUlki)_0RxA}Nx7i{D^Rm5*BK`{gvZ!fN-Wd$>wC`G1kRzz; z!)Fs#=I#q}K>Q(M^e4Iv>IZ4Fvmtp>_PA%Rd;ScWYtbvB&x**RQP*?Ej@`dba+ z?Zq6%J4WFJnJVd_cr{iPIKXQw0n)SBGo?8d9&*s*N|Ris#M8uqm}q#>uUPLUNyoXk zR!VV$H%+&3L1di0?8JRNqST+ktE12h>pjxTW|BpANedXviqlGEBaRl9B)Uw|^UjuX zNRMJgb8D=F&2p?+&om1^VcXSm6=lfIBx!%_f5C10Ac96&Xla~~Ak#|Sp*q8t9Vujj z+tz1CRSAPiNzn9f)L-<(+gM_aGs?-G0{BjhYIx%hJiK|->U;$O=bJ?$vwota?_L$Z za`O59`J84OD#5*|SSSK_vZa%FlhR$(1H)ies%M&rW9A6dvPd2OZucw;>Z-&=zz8MS zN7mvzwzdUKqrzYG1HO#L|Izhd;YQ+0!doBFQ>QZM6@YCpP;m@wcK^t&+=YdAVjjJ5 zdg#=E(4PA;qGKmpdq0(07PRZN+W$!FiQUIt>Teir$$Y9=K2m1+Pi2w6<7~6o65W|l zK%oF;f6yVg$ ztJTqd7l3$mvJS;WiKcc#6a{F;)Tal^qs{+ijTUDKwnj6smlrh`B_3-?^^d>RHN-_` z+PgzPuW;A6Pm&+UVTYckryY)^qRyVw(uUr%r}So(b;dCb@m2cMc5_=KKz;;UktVkU zefKa`s(Mw!VKX*Us$$2ljNXvgZP{r>f?rk#Z#>yW1+n8Fc8tK(?|0)8k9jdD%zq2B z-_cgoQgJL1_^1(xopyRW`YZO|-CyABtf#5QN;aCPbmjL|=MPHV)aTZ$!7p)N&Ffx2K1dfcPjp5kD8C%! z0T%pcxcOaqXV!H|OS>E7Hjm#Hd1$v~bT2?;#3^#Y&!bv4--@Y)(X)Wi^m++}%7bKs zz0kl2t}BoZ*pxx(gWjV(POZ}g|(?0l`yv&uTg=CAg9u74F&N48`R zDmvsU?$zY;9cS+}%*MyjkOIfxKB9FmcK*iVepOG)?Vr*-w@wVJJGTsM@7cHo0f|1K zcdo6QC7eoa)hf{OzH$!8dJ-9J6rI_~tCFp)>A8D3xn2)Dwn|J(djAq&|JvPNiVf#O zeO{YF@SxG=`~j}Rkd+gIgxmD2-c-LDXW25&_#D0Go1tTov>ONFx_QE5z#p-Lb-b@S zJqK3LJrwXZY9s5C`L~YX75%t>9;04s8v2FW7_cPad1p86a+at_#Urym7PAbPIUn}_ z&d^TC_AVgK84*iJSM|c=Ix;721;)9JWiz1HSYiY~v0ma04erKPj$ZlPvGj}4_6bg8 z+!u}{yYED04+*t*a!MD?QINWQ`krs&e4tA4AJY*Gn>PFm=XeT8eY-qyC9GNkZb|D* zyj9%p@7yfc4Y42JH1a$W`PRxigmnYl5^X(4WQ?}8If=0UT}>ZEwl^O=@@(hVxM#Hj zh^*bWwHxebokOl-;z!UE2*(qhAG&_`R_cV1-MsCFFRbVs)ly_Zwr+ux0<*8v+k+Fq zxQPln-Wy4f;F0n8o5rbP6+;+nco1x1?=DwEBZgbIj;;qWu-+r^45L-NEiJN*2gX;f z@t_}+0RQL`!RshMuIPa<#p0@#+rXm1*oeQG^TBKDUhabpC0~8TU+)~ zC~ROS;lS$`uCxxBwALrUvROFR?ly0>HpC@?Z!~UCpZcfr_JLHBpH-IA74bkHSs%b%YG%F#}NL~8>#eSa4R-V8OY zob!Qc(v!sy$>@Ux=L<>>z5XJVONC*GWNawd_?#%tPgKVC%`Xz!%^ z;6fCb+u*GM;Ck(uqAI+7jnf!!bY z=zji#nHZFTj3yu_VQ0Q^yz3MQMfEaPxf!1uw5VA)oB6<;?SEw&hjyz6`<8m55`yeU zZ&jm8B#9D1>T^2!yD2-(brVfhahEI2D!h8S9C0VF27D1Rbt}(3!AUufb#+aX`~BAv zxU-rEGwY%*pOJ|W?E*!c99b2fD;*SZm_Hqzuw$st5V4;V;LDZl>kw(cA>4AT*80wH zXFHZG%Qp$NJv_hUFE2f}YAWt=>u{TB8SQEQoza-~jCOO2-e&Njg7EQD!dsJ|L%Eu)IAd^e>Rw zXV==lYFbw23v7K&POR&YITzSz7S@2IU$+5x?_9YiJuI9?&{Tfo0hAi@;44#P@H@j9 z`kSYUo7U^;MLPicdZuy)E0PHRuc#+f7DrK>?pX0ZfZxn zQs8#nGd;X_*lE< zw6|QnYv+?(cXx%|jFpC^h3`=&)?p0+a5oGfJav5G84QFZ7v5B zwXg$F3>UM2!!4$V4`PpMK!KRUJ6;}`~-bG;db4cWX{UDoHnpW{x@>QX?fW{ z`1{P-1J^FEko_rJgNNzj&!U>V6=UcyxV0F00{D$0Td-Jnf{w&gQD3Y8`qnX2pwdrG zPiRXkqpzR6Dd>JiK(Grakm5u|ji8g%VN%Xf}Xhm}4<_qOa)*0-1tAnAbTHTykBDE>Jp>scRb@G&2 z&|jZgF?*o-r%23p*$(D3-tv+lHSfBPTF!mBC*?3gUg5Lsp;&jFjbiIKFD4c7{-?;^ z@V5<&_HDyW;`HG0If}Q~cb}BQ0QW+4$li$}yoLGUd*+3$?u9k;ZB*Weah@n|iGVbj z3gqw;&xKRSvXRqLXy_Y(?w!syYx7C1ulV2Ru*FoZj6GtxTKA5^)W=o9!f?LEB>=)+d(=S z?OJ8^%yykVrb|rfJ-r6Es;S=Y;k(j^@!_4^+=(Z=shN|Pf1#!09td70-qY5vUI?mG z*u@Ekj$mz;bn!tSK$u|rjE>}_u;8h>czEbfD5oMDedTG8j{#{{2Cp- z-;45Om4)q?jp=3;2W12{Z)rkepTWAzW+M9>l+i2A31yFGywE?~`tMSd+0KBFy>S4s z5Lq9OS;CB<@*i>21V`fi9tH=fL$N$V{?`nm)U7curR$%1yi|B>ZYj^xRHKm9=&0sM zHu+f0{k;VQeqpfg(#`~vLuYPjXHt?{ILob?i=AAacADFwms!q?*yxGQ&q=!oxgHpd zjM$u^$6cE3i#^YXBnRpMFDnULW7>ODhwYna9)_SWkSYd)e|SO%zTYn!h(XiU{oSX$ zzt6**1~#(At1Oud&ZA^1bd%9~1m9~aS)uM|A*9-1i?$Jv0^QfHZ z#Caep#5YFZ`**+5RUxtHlhT?3^(XhK^1_7$OYJ1|f*wk`m*#^7p^r}$poe3h_h+*A zbM%+|;6b5>==bY^_orK*%bx!s&t9PC-k>~PAa}g&cf6$eJVg7vZj-(5KHXNmOZD^$ zgKrdkg344-Ze70+3t3p8y}3p(_Lp6wOSG-~xEtg>BBUL#_&h`JaDPSGUHJF*Cefsh z_4UnJw&U&Y+~*~kA@W7v=k@g8+rxf`<_?eTj`|C7e20(jRZ)?KOIil}&!5?o4*x8M zF#xAeDkAak>G?z1$Cif#G@sMH2+>8mM_$YOn@;LbMrQ6w#|6);_E+Cw$*27(kZ=9L zw@2^)El_^wFr^G2r7Xx4crx@IsO2Fw7B3%p8t)Fu@-)nY7z*8&A*abY)*r7T4>^Wq zlPFyn6Ue+Kc&oaeCNf7Z-gQXET|)X3b&|B7cW}x^{$$;U zp>J%z!(e?VxiyEOt$4_e(ef@{D39nR7dPPWbAd+eKB#K(#vb>)b3sfA@7zyErSc}X z>7~SuxzwkTz72um(X}kjJyrX;a5_cH+WHdtw-9aDl9n~C0YqjJ_W=4~@(gcCaLxEVbUAnlDj69LE%Lq8n zWe`Tu_;&XjG>YHTL1a{p8BVc+ZdqNlYG~AtN7Nola(}A}+YD>t2hYOn|EMEtIh5_D ze;&jCwrk5!M@GfDb)3z&Y#pz%7RP54ZHCS{BsAaVaBReAe6Fd>lz0 zu;@J_?Z(;Mr0p`H>FS|4tFCj#ZiV0aG7|E=r-wDzRRJ4Ko^L-6x+yj(=ISBaI2SXk zq(AfL2hOrl*eV4%f|P?DWDZXWIa+Mx{W)j`Frz;aKM9Qm*`}w+RFD?gWt?ZYeEOwJ z>3%hIfpr_x=9mJv%C^=vnBVQKRJ_DT%;S=W*tbjgCE&2dZQyZ_H3kqz#!Xn{Gr&KQ z`XxKd-DOvlm;JD1pC{oecc5q#y=9EOO)xoU>I+=jZ=E+ilsHF=9&L4spr$FlqcM0} z;i?KZM_hNGN|n6(Hyouj<=;yyGe5K(k-?Cc7_TJ{zKPNsIb$u4=%JR8{7fIM(K89? zUeQ-wCFMa|{jlKXn&9fc4pMu$*7O|wN{K@ z!SHn77A%?ljWwz+gHm&YuF0$%Sl-rRYx~pKZ+}GB<0lA;OBMkA5bP>ral5s^8AXc2D#hk7cF)Tlu-u!ta(6k~hVU?iN!H?L0ypE|xGeAJ11#hSc&DV!bj)qGYbb&vjbM@d?%^Qm{*VO>6dHnmv_7%uoq;xLMoUhQgdFsw+5p;7J zPb5X$^bwR6QRg>m;PadhRQeW@Gi9qxj=Dzz6#RK}8?Ss=6oy7XiE>gvQ$qm@wQJHh z=CUixdo;1_ntj|#fyP{K{P>6I4$K^Aa?$y*?HIg7Gr&d`G*1G^?#49q7r3-uzl%ch zgZ6X6q6EiO51X~7$BJ@`AYHhGXk})?5VLumkAJ0{5E<%jHaRgshG;iC6j~w8o;;p@ z3v0DIMRlz8OUk3eCn*wXdo-`zB7twxxZu%;adwUX*IJf19sFhk1 zDv|MN3?8F`U9O-?JpnNN@~x-5hOcvdM!H3Ck@Qy;@&=+3Xu=D>2;(43=LP>mR*c2b z3X9AbKpksep)ylt9V+jk+02thMDub_rt|4`1VYi&=gL$&A?T3%${YT6yJ=bRVQ;yTaTrDmBsH4n_8RxXggPDdh#9LaG8Vi zoS@J*X50A-YRQ)wgj#!o$<&CbeiWMJ-|`n~w+Gj<=w{Ek+y^Xu9d zQJ%2AfT)wG06Rlu1R?s5%b3iL(Vyy0h86ulq$Tyb!lK#RwEPTtA~Tu4B#I8qBboQ& zv|O5|mY~6PjYZ|+T3I~y_pSxVDLs-T6d8`hrE_2}SA2?i#!rxIW4HS6rk6iX z#C_^i>#FVkWYi&Db*A^-@|M@6*k1-_s_>+OG;86TMSeJDO!&Y*BKE~Z7`JgccaJn* zH`9k>&Zr#(5pgJ7+VoXT@t=MEaC``n1h{=m?1XoOKFtI^1AOn$JFk;;3`XtPmPBAK zLcdZUSw|8^U$KjJBywH;2c#TzG4?I7&nNm3ti2(N`b^MWk|yI)@4B+ zv%H(5Sen!U|B*nkTpmSr>Zjw{7;8U+tow33KW^SsIthkIuTh@ze?E|L%@U3`8`?M^*<@qzXBv$kN>m(7ce%h#%b<;2R%159GjpA5obX7dD; zOA!Ot2_U62o~Od%TJ4l1;oWmf+a;y0Ou(Ig$9q)9rFoz-|CP41?I$KCH|=nkvyag$ z$`qoe?&=LDbR4jHY4N}^Oqk$-9cz5pCXq?nb7e?}-`$)6sb3oq3K+qXXBSvsN9<3A z2FD0;?OO7egB1t&W){Q}eYzH!4kVAMVhtA@tMN#-^1rHL+i)|a#wQxt)myXAL1vGL zp8P+?buPyj8h3m2D4|AwJ>%Td%Y(%MqGmO}86tZXVbqEy1iyIcbj2vNGAL>L$}dFE zDZ@Wd*7OXYtN=Q@{jth{NW`%~e*I2=_FGcF66!IRsXyv8Js{Eh$SXL~-AlHr%{RBX zsBsY78=T{}IC2{{gTN}F-{P{?gh~IYzSJ}Snp7rWp15_1UYs~ek2@pOkd;Yvtd?BLGGJ1sNHcJO(n{rdQ2l6LUx(w)GyOB;YqgCO?7CkhOk z@vzb0enDvq|0{{|O5&m*mN3zYL5pdwAQwMAYCr_+yxtlqXQqo?oZ#N7}%;E2>4>u-nTl843Y-V`?RlgQWGgP4_a=8L$4MS++Oh2jQ-{EvrzsBqzv z{h(IhP3QQiwghi}>uprd6=xK$jqqC*fR{COZ@8^xIu}35#_Hlv6pU`ou`M|EgR|1Q zX@B_`y8k`cElb#P!D(1CO+Mdgx6H{_1~Y{$4w&{e-^pI9qQ_x~p<>>k2>1lU zp*~vu0qBuw_@*?RgG2<)DfUXY97;O*_jp;7yE#!tcW20TH9+7}|5(q(@Ww`*_0?4N zmML27AmL6`mM>5sJYz-`{Gk~&5a@Q52!5~gV-sLAu} z5To|$!A02mx6=ar1H-~~O(SyFa`$bk6)QMP-Q>Dc4`0-{PtP-GFxaX6-!uT$w>tE= z0QUM&S;Dc3j=wzv@Ts*n^D4FwGZTdPp*6f#vbbw-cPF^(;_mM5{@>rdxw^W#s(Uj%Gqu${Jv+P8-Ou;)eNxoP54Ntg zSET5j_et4WS=?s$C*Z~gW7g`z{dH%?76)Rp?i`?52ymhr6_sspNRRpbQdhK@-(~dt zr%PYhOdhSk=XHJ<>nUQ_yOFti1tHsON*Hgg-uXBXG35S=1hwp1JRpaIgxPXxD>Y#_ zRrB~#cmfEa)x2|CG##?-VJi`EVL01AQSe>-YXl9oZf*nay~O9LaMR16x&|!sGIpr! zX?=4m+z(&Ql?SYkvk46y)I=@ysJ2~7;&iv-x80?C$t{j4gZ8CEoP=9T&d*Bo z@$TK$joH+sMbOuxE&7j2mfjHN9lStR9RNcjazh6e){j1NlxEzCD&6P0Wh(dUZxL~dj(%FXR^zY5r!Dd`c4O;Bcmq| zEGe$IUnjRR5<}Q=G4W_4nsXwA}HMz-(tSu-eRGJL1zRR zlK#Beijv+niPG`jFhK=jSQpFdJd8+?3_T|{niH8i-mEWWnl(SpS%Y)I9#@fFwiHp= zAX{Zq9y?k;f@^w(9r4LpspOlIM*oOz+9kP~rc41MO#zITn?*hV3MP?rbo#p^YMFg) zoqc||!C8OfR={1GR$}rrwZN@UtxxMiY4G6_d)}cb;MT$#S&o0eems%@Mum6V&!IN zs&0S=;*hRt$8;%-F(YkA^)*tHL;m)k@F;xo!Q}Id0Z7=-gt}}XmcA#W8(_Dt5@LY< z6U1z7UTI;x%)7&deYEO=sYkj)iTNN1C4A6YUsxI7ahBh$dy*JQ|Yk_vA5(GL0)oJ{w=OE9arV+L7W zj}ljE*hFk00iCzyOL6j5NPx%)mnU&bYBh1oP;V21l3| zbjijs!M){=IUF5#ifb*0AR>-{prpQXRvQyTB_!1D z!TRs3evIT_<~wbl$dBtkX3xGVgZRmzmW1FXo9oDQr@+;UVPB zcKvv`(!YEdL+JHS{cEr=o>Ehjdtnu1jr}@ub6|GmtOemTR&&icHWsf6Qh|dxoYbr^ z{{}SiYeVgmU+YkiH$D#OCsGS#jrivs&aRvi2jx$J7`6;@mQ{!*YGzYA5IclV8sy5l zahJm1vs0$AB%fRh-Wb@(Fh>cDUitF3h0pVcf8aFomQSTpK%Vdsonk)l@l$mc!R&cE zc2F}YZ(1uhc|_0J6Mq`-q|Jm8Br>S;)Ird@{`M#tDYW{ifb1qO&*8c4KBLuSudhQ6H2LtLtB?Fk7hBG{M5SQ zpA7xU&brDX2e4RP_<1e^yW<}M@8%XTX8R6H`bi&vQJE{vYOlb9RVVs}+`%xZdTqX- zT!i(=Vtx9%1!5#&eR9_IXA*8&Mvn6}oWfO3;EQ6u#{mo{t@zZe({*+tI8NYEtixf!M4I0S(hEFICYoD8o;y6}%uxiX+_7>%H#~Qqa=lT^IwfMl3k2 z4hSFD4jWi^QXIpQn!@?2zhn`+v<-D!5n5K+<9yWk&Bl8&wd5a>->SURz0`Sd?q=m* z1*NT1rcYBH{TUxL=1$!@#uN@Y%;7V}S zMq}~C5N4^&F{6syJ>Ao8g!os{x%cyl=;$^qUt@|DRozPzp9->(^F|F<<*)H`o_mAJ z;yJqDsdnG%38giVcZMh*$orJH0=kwa)F*yKLUJLBR!Q+_%ewZO0d>!5Gzt%_!Nf_pc2DPN{Gr`n)d(uF6LtO?DAS%$;mnw|Bf-A?gwYPD#oXHPTN_jIu+;ADnG6q&{0Iarqr z4^*o*!2C#nGyl)9hUL#cJes8@QkPp?>NOsIZL1K(Ty94(ObUk3)~h^Xub)Jcd1&>) zY0}9PZB}qgR^lv^(;k=eU=aUKVtc~`q1ZTI*(y?ff>DOFscO-`vuXC^U7UinW}C2H zfY@0!8ZQpT&;4ItW@1qjz$d^QR&RScw*uh+RtCX5Y3m*r>-Vf&+0YE>0*~$v>vOJM z|lS2{ZF+bK#U&9`nF(ZdrFCiDG^5->G0^q;&`zG$0CK zySn$bg2M$S#WH1;y)^g=l7=WC|30UnLE#v}GSiw*MQ9l0s)9g1FiYpmGWTHzyR?6b z>-1C@8X;gHRJ{o&k5c#ICilU8tyC*d+cIgOU|g*ml1Wu;$^nEwfdz!n%T;^CLrAG`N1z2c_;cfk2OZx=+S? zl9*C3+cDIxmWHLOCN#8-mR0zIjVH6;5~lnIm>jXgYIgG5jynPq4i5Ql&uh1$<9k|YmdiwR@GRADjf89F7}#euKgXyJSJv(D#HWcxhzAw?3Tr-l zfVD3%KSTm3Z2iTsI-+Tr%ttAEP{%f#EycoeHYW5TdbU|TG6c-P)9ZNoR%QYKwbyr$ zD9^?hboxFWE>e6HJP=a!>K=WQi6r`_%BT$!t`v!5YEXh1pTjdp*gD4u_gS?S=R9-He@C5nWZq&#f_jjY!f|EPck=)&61;N(zljf{&rXGWWI3TUprR?yGwJYr6Ag%_{(h zl6e4rKZ~Vrm_z+`lzozVGAkN_kjn6i^B1&ZP7CE)aM{YrY5l*fU@s`By7V{Pvk2{h zwW_IiWenz*+!)*>lD=|1@!_Xe0T`YhP6{XuW@wHVyOZsDacbKOxCM*&GeI) z(X8p8xd`67Cov{QGehRG@+ocO<`q!UwefBi`NxOlYptya*1*d6$lx@-fRPE+$ePpe1eNiYv0&Z$l{Jt0 zha`_uaIOTM$7x^a=P9V@R;i#Hx7*lJ3$})wt6EVi!F4adD@W)f6Y$=bA{X zY?@cQ(6;i4J;|7F`$}q8l=j3$8RvWf65*PldX7oD#v+CypK!rQi=2>Fg ziq|!$!C%c9cp@9Ux-L*>|j=yhfR*0fephlSzfhR`#Lz}=P2%ILP?%hbOk?Nrc>*QvPe z{4!RRA2adv$U}`Bs&H)OR;d`0=r+6V{j|Yvh=@*_jJ7|AT=i(~ZS!w!SNXNR9W~^h z>~*hChod0gwViSYe!^CTj0^vULRRG9Pvj{2u@s&T-#h}|d{EmzV)PWC{0I|UG|Xaj zUXO^MlB8;@xh|MSdZR@kZZGz~W&acy{C3Ui61$;?>vI3h2+E|ao}$A4c2ul1+LRon z{rP>l?NmaWUlG1AX`@_;TM1s?*}zD=v{n1|F@LJ%i#z!gWC0n0offpT9VXvhSZXy9 zv=2Y?>m%x>!o5dXA^q<9@5`B@!gz%n03QG8Z+KGor%?s0%JGLIkZW^_-xm0fuzwjf(1U8^OBhYN-r}%c7*Bmh3(%%O5)8? zCc7&^_(R_Vk5$MvpHBOJR4t^ahZOsKI%SY5zxD=%xmjQw@7v1m4Y>y~&WcP{Ofws?*d#1~!L+>EtXgB_eOHWjwdTC$=v zqB2gPaZp!^L*A2nPs1!aB0q7N~)|kB4i^tvXOscjCoAl zr<&30PE&p5*MKf6%^3{<$yu+dXl(9ODFByMt{4?3t#11Ra;sJs=6|rfJQfM}?x$Ke zrB!W$gW3+uS|%(D^1fd`7DjRMzuR$~eWmRlJ6N2CIc;0P+BKpWwMi!kd&4Xvvdv8HcBJGV4{$TjU$}qZ`gU<}y;@Jnc`m^IAzkYt62v@`d z%_cdpi`^dx9mI!qyD<9^wKch`w(v;tTm;oZfY)|Mr@d9eZ;hFD$YX8!3hEDk2c~e} zLGV6H3JC95F3Rx@HNb6`b@Rubn$l)Otz6gW9sG+Qohyh17_Wr`Ps>L1nT-Cc^tM4- z6Um}48yB<2FBsFcYS5T36bG=#o&Oj=zugr@je31&PTPjV)scOGGDu;ZJq37*UNm;f zTAz6_8bFnw0k?`mD%#P_J(<0>|IpYgi~((W=7HYX{7qoxvWbc}dZ->V!Nu`4<6D#_ z$C?YSIF6rPAk(~nwEzWhkO&G3vsY@XqK%;qB1|{oJ%(cOSJL59jf|_Svig~8HD*-sf zo(#qJk^9vgZxxqDc86M9gX^J9J<(oG%|KMEdpywRfAuO~=hx{6W*-0yL5hB6cL`B0 zEk!28i#*-7Pltqp_aiLLed~0a2Bu+ZW|W(W0uw>D1q!Mr^B^Of*Ufwh-nx5bq|~j6 z82U`$n)-zeLrFa}i&9w=>S6w`_DvL?t9$>IrpLRHm1z8158Tx?d4`Q%QT@z<-ZGj& zg-q7UISb=F1zq>Q%26g3MXx#?gB{+d3CXO?i=X--%o0ZJ#Lmh=7yZP-yYklPeHYYe zmhQNSXrS*iX($alI_CF76PiKd(qj`H%=>hBSGc3fo+Fur3z`f(5Ps2W;&VED=kLsJ zoT`lhg|J}R@dAedSIj?VXL56%W@m4e)k`udh-&Qz{&?W!;c~h-v11ftRj}DCLk`MV2hGbdOOE@;kJ{0bNhE56 zSggzo$z?loFg;?k11_|r7nGNXaLtXQUdT`^DHS89WD|2&qfbA90XSnZ?R72~O3AEx zj>u+1>sw#@XSrr=_4ass5C&IXP_XT}!@KCsU9GEDLJdWiX*66^+x+2%X16a3N!tvk8L$($jVU6JB=5B{Yc zQwPi?+jD8s1-OZI-xIGBq~3n=bH#xdsc?Sm#Yb^fSLXd*35U}019jdSNm%nCe zEN_sU(+SGh?OS^|M}`Rs*XWxeqUgWvTJ~({245)8MTl9^O9G|n(fu|<&2Ck2`uDkX z57V{zlD2w5+CqJ%IQ**`gnd6qkcu@Lb5r5S&T$7e{Xt^S5mWF&ljbq^Mzbk@xpxiU zIkC7$-dGoAIF{BKlmV%mmg2z`4EB(?_0y3BCeU?&#KxlW)p0&5%5kt;{_o+O<`|(7 zu0f5PC)g{oB{a*cB*3uXM zSWqX8kM&OR=LqMm1OgnPh^Pa$dDTUA6;{YF=85?rb@2cr=sQnxXI&@8Q>$tiuNL?c z+YC$ww7|F}#b5(-MBdkW$Z*-s$Mc~)N6Vh9Ksgp^7_51tG{2>MxYeB{ykbFrJ6l44 zOkOoV^3pgI4atEw$L`6#l@BCoCZ@G zvon2TsV5eO`*dY%6}e}8jBmD>Hl2Tb&BlHTc~-4cnBF6b`E+XXuAG$CAJp~Nd3}7E zNTJU<^~6@QYjl*LPp4OSCyR@x3{q9)@uLZQTxOxzg}ER5Y`VxAJ79p} z#**`=o&|AhO{Q5+!i%IW9o~H@SB#iBV2_3! zP&%wc^3*0@Ooylc?CA9auw>pbo1JP?VC~&Jxa3SHy^sW7c9YOvuu7B4BWJTy{LEtG zoK@TOIauL9`CEtyw%7p^?uroif=I@q(v%iUL|G?mZ4=H4Z$^rxp1^EU%a?Hudry9j zH@rr#Bo+sk&s!puuC{uLDZF*>tzKR7Ec}4zvScyt69txz$a&uH6!?S;GuP8YZqB(O zCQ15BkFhDY%`!~!M#=%%D(D2Z5Le3L>|oppEtr z&NDx?nk=h9M*Y2xrN!+>R5UHtlz1JHSuGh9RuNxC$zga(e;d8_MAaj2t1ngD(}Ry} zM^CF|cv^q+uuEm|e@3^l)l*W z6;kX(TCvU6Ui=W^a~65vqjHKKrT`HNkn;}2gMv`tdf;#q<-&<0oBe3i66WX9JW&*W z)do$qX+5(lH;SqcMS`X}htlZ{g%_;e6KHOsEoSPQH7&Ueg9*~Xv{dad)uOwct+>V- zL7Mo#$o}*)*?ZzE2W0GrgD$;$S@$D+=_X0cXO-g4YeD4zsnIx*8?-f9!eo9Cckpp)W)qh>%hCh8*&No9&zIQ&RjW@~`Rh z`^&Uz1GWMrDYm)%8}s9MAmy_iWZV-9{%hoJh>(oCJtyAQ@W)2dDBd_0ARoZyW`^IT zfGu-AR3v00H>iyOoc|l2&AJi3Io2G?@~5f{w~b8LJOQ8Qh{8KTivd74xc@b^o+ZiX z23jjm8W&T)MO75XlaUjyR}}kSn2L#~NYqUM;Xe7`)91zsH38#dYa*1MPja@SID#qP z|9mesRd=4#uW$12w#WBbHqZC^!h}DV&5jEXJ_#pS>y@g@yUrRU7^u~N1#uRSLe~892#tBFP!?K;FHCj*VoeH}A{1D9C@!{P(rI(#P&Y zyMxc;5~&f%3bu0VxN{AIe4#EfQ))27d8DJJUwlCxP1S9S)WUT2Bh zPq-Ot*x4k8R=>KYOJgSeB#54Kd7&bEa{eLP6wMr`=gC#1ND|F_61^gaj<7JsF9sgb z((;NtrS~BtjBlRObbWvBerCMC0SCh# zgf5*384M4J1$xaP-VVp;;Cw!Fc9_w5S+B;tLPsXW^%oodY8yl{i`0Vk8*P7+KS+Yp zWPIJgjF>5o4}PAug8A3|pRVAPe!?>z1!zOc?lt=Sd~kyF3%akjf-IayVw~Wih_-X0 z4T8xFH-}EJ{$Z)whmxYSrk6d0gQP94e%@R@Mk*Z-EpeK1Px;nAXQ*IJ5iYKl*L{r; z!8v{86(l;lNg(9q;+XOYn1lYlqLV$==HH3>qww4bx*cl zI*|xpbG-)#&o}$ofcEDVZ5n1F-UW}yJetM2)L+^z`X=@Q${_^5ojHdtssp?SXZ)-C z14zZ+GNjFJlT@N1w!4WZ?)4sZB%c*cASrVZJAw_(0wV-+q;~ zILT(aX?@*aHbNPFu)9zj`zahn!g)gDn$aDL3d}(=bA0rJ@3yB1$#(HPJ1ZMA75lz$ zD=nK1Z|1l!RNFf%y$041o;>v6TkvOrEbXdZlyh2S3pcWU0r`V!vUxLj%Ye9#T}n5* z@rz$|+i8aeB#^raU~v)hLhxOD`-l6z`q5r9GCa4eVVJP|#cW{YH8ZknriMcMo&R+~ zmZvRO86XIxMeXE*{~zG}ZoK4Ǒ=H=6uGM>Xvl&Q}RM7rRbY0n>~^JJJV%Y##*X zH9h;Zz9asn{I$Tma_ec^T=w<ZU;zhUh`^(0>;t1^A44F%|I=y2^~kJjCUkA@nrZJ50)B% zIDXYLFD~%oo*Eeu+i`SrXULAsK|DqFjLJOlRY2TT;bZq#f-56qm@DjACf`x|d|*J8 zVwrQC&xlKEKZ@&I#9-@>*91h~%zi+kQk}=4&!c5|rR!S}eA-!D#}mYRHux%dd6nlY zjTx~HxP}bt-d9+)^E*^bA(kDPw|~Z0&3Bh?GSe)wRg~+~OH!kN$B=^!6Q}SA zvdjaGg7C&f28#=7MN#3CsYDj=fRoMAPS!b{J@Rny}%Q!u6(uhjx;g`17h5MMlp)FYyG*_FXf{F{4$HDzKUclWv zsZHP7gxZj;G;}U2{xvz3&DhnCT=arTlm##?Umm;h8 zBZ1N<_R7)F+j|4zT-m}ngbN6R=P1{_SrjSHI5O;lg|LY4n2e(d#BLl3xV3zX88gKh z*VhdD?3^b=QCw^D+fj=d`A#Nn=tfi6rE zmx0qP0+d+*4uxKECwHH>dqZ0m-AtqQ!{h-g=i3*Yvrr1+vB=lQYA?+0%@ zE5<^A^sHrXjx5W`3N`oL#W2HRHf|xQh?VsxtMWnjkQ1+x zXNOoB6*x3dYfRa{bk(ncNm{XbWs>5Hg!0QCcY__A{H<4J2ajUeUl<7|aUdkD@h{~r zI`k@QloRgj`(4c6sGbMRs||DI8GJFaf4d@DvB)RvMAhF6G6FW)(_ z!Hli9!#Y-p0$|H3s{QTJb@zW877PXLOOIxFgM_&S`n_g`m+jgYR1;EkL;jX%wcHW! z86VfxzDVcxbX;kM8R3C|Y105cBHTNq{eNDe!b+1jWrTEA@=6+dS0FL|HF~&t<=*z$?I;#+* zgm|MAs<&TM*92QHHLm`*Dqa632vndHSsA#+6e>Y@)HR@M`Yo=^>lwBzTl`!0D34@) zaDBW{;@GMxO(Y*yn!7_)sSWm1TaL90+F10*++O2Nx>D7suVwj9o=#AUMMS69>K5+Cdj<@Kn*X1sY?`eL>u*ORs z;`t6x(72Z2aOhwP#uc}KQOM_PZpGC_2NCxU0Ya#@Ij=R)WUxdt9Dm0I6X1Y_F!&(R z#iZY!7(-zM7&AK{EKlw2$Om0;Gmu94nFMqTVQsC*$H zU%_qRfntd{1e+1#aQ4gNb#guX^wxN)%NX4FDQailMCi2IjuG_-r z3hYtm4t|7J$zufG)u@yf5hC7Il+oy-RU@v7TSmLeJ?aNl80|}{bLtG|*ECP&X4uyJ zScpJ{A%`Mskhz-v>9zT!lEJ3&)9rq)IN9(_`Rb-c;|!n=X%(Sq=EgiG`HOfJnOUVA zig*v@Ql6h102Zwy(z$XD}58k{$_s|36$Z0<01R0yPog!F_ z)PJihQO`KWKeGolJodK~u~=KZVIi^=H5q3kar#a)Iq!~HE>x%NcB3H}R(XVJE`bKR z+h~qII)_9DlEpT2m=6O-_4oLzwCWDtn*8(c@e#0mzV$<1!Zdte#b{<1j1qWRM?&6k z6^FSqw9D*p(TA7&zQ%XXNaXFFU)E2V%e*^YUA#(=QCE2!@qk7&wfeas-#I=sjw1xp z$N6pGqT^`cyK>Qil+%@)H~ojKUDDM7NseD$e~)uSseY4>XM(1o;+=D02F8*|9CLey z*9r(pI<$2vDpU4%kUvUFG6EfXcPb%K?BEn8;JSA>s(vu0LY?uUSd5reUbJT^d{OmB zCIBc8Y$Fv0r%0c2$i_K+ZkeRP?+cpz?!Xp*uoc2-QEW3B@_mitJ^B=_Hz=7m$y|IG zLU{bbi2z;i2$x|7gZb7h5eVp!pQ@6?RjFVE&+q?BFXi|x=)oYOvGUjkD zXW%ej4P-`J9~q?izGY2idFPPp2ViPe-C(sob?8ZfOY*`G3k+6?I;{a zTQx&rM}$_#ziPV=<(spLUc_gy-0<4Pt|tPg7i{rGa~TpU1)odcwm4P}y6D39Gfxt$ zXiwQK5(ws%S(kgvgS%4-2e7kJtVH~~+^IX*yeaS2y}Bu?1;moG+vFLdOar;`AUfzv zpN~*1;RmlGX!h4U1<8QF7wMf&iVRk$=YwC^SSwTh{Q5``w)gJv8J(sor10WP^Qtla zn4Mf4j5YXd9!~{ongYV?%T&B&p5QTyPaf*lMw@<21B3RfWmU@oA-w)B4xxRzSVHVv zPbdDP1nr9t&XS`c5#|}J+@hStEy}gsI^c;F#zu7y2a`~L>)#Ka%wM8*q=vPy%3~oD z2{yD_Z5lGDG%tj6?L&7m^9~a<`?Y^w_=%+Wvz*J*p9%|&<@mp=?3rQr=V+vUA)Ii= zJh|Uh8YGCT-n$95K5gMl{7Yb_TZG$tOI-w1rh$^lPU6D>4>aVtzuHgaPoiY0h`XrQ zNkA>StE%U`r%qu5Xie=_6o6@|pEt)7R1tguJ%so%fE9yh`%aAm3b`3d>~yjJ;S-_-)7mJO0HE%CUCyC3|OSGQ%vk>IyXMF7)RNsm~^(6OyVA(scC2`r)=k8FgG&sySUj!&&hV{D*xXk72V|w@cnnt=8Z*^2 zuyDRKSNtQQU=Uu9C^OCtBo=P%_@$ zpi`;np8udV?{JQgJbAZZ2G{p>0N7x zZGo4YS&lDOj&No?sOpbN7rMU4GGeZKpE@;4=Q_`f0E|FS**|3JJf zY;6B$65I-}lTwraAl^5OyU%#R|NxMuX?Q2*iyJe zo(>ao1h8p7Iezc25JLEo<6-nb&*vjq;I&_p@pfm~=k0Cd<53P=@S6SKZkEp-gO?$~ zm9GWTK!}{hQC~c^%q0<))P{lM9@Eo~C-=ZeP`3zfME7H-W6HHfe-pugC~W+1xo8SL z_}c_%x##%q-N)e?Pk?v$NBw@Dq#QNT^VTobcS=~Lh_3ke9{seJFGSJqe=@4(`MHwy zuh)Bvg4OBP1M;}&TC{*L$pRz;PLY_{kwskPErdqSC`tls{rXX&%=7KEKlDf@c^CF& zI}D%mH{%)#Nv2*Ch%_$(Hm;aW#SQOgyd{S;6JY9XbCe?!Ha_DNoPh z-!glP-_DdW{H7X^IrL%fu9hNH!pD!1o)a zlaTu22F)6-Ul?lRHFH%VqzyeGhIJYE$cfw3g+7khxZ%CkIcQ(<_N>Pp5su=8ZGLFw zJ00d17aHe_u&NEH?J=Dxc`=c*nP^rlk1H`l3T0VP8Zpip?^3^d!3?$hxQSd0VK5~R z>vMA0zUkL3MX3vVH~A9|_tYl%*k)I8L+MQkQaz;;Kt?DO!Fc1ecAe2IPP z)Vz=TZn?edqs_=mUi5P z0{qN~{^O2*r?-}xa^(A{XC^m84EdglH5G|Mtwk;Rn(9Gc$`@udw?V=Dr;tf;>Q)3VkOlfNgxE&b9 zimqcvZnpQF&>nU5ws)w0Arxd8rf?VN){OXFn<_o_W4yaJkH;VoZ)@0|Qea@4_KTa- z)n9Qpy6@|n;Q2II&6$`o>JXA{tRB^2S{rX*P$M3{9=K9j6}@^d}Yrq|KNB1Tw%#pzr-O= z3~b+^li?3p*1}9}S_beqe*+y|9q;~T4ZqmG?b>xY!gvB3gzg@faj6!V4BBM(d*oHj zw8;Vj^v0AiSk}l=SNDuFQ4{fN)}`uQ^hqnWiZ046*LUqcH0KPdSKk{v+zKSJRLuv$ zTSaQ&tg6mM1IBSuiq8)3v+~n;WF2W`NvuUOM!&EHSu}Wn<*$0WZNFg2+GMV0hI9B_ zN*)@iW0vtE3ip1eKN8>`BnE;l1iyL{#~;r6?*aAu{7xC6AN+>5k5gP!qrVb*gZ2%m z20f7a?Zl4gh9Pqrc)`@b#IWamNY{Syhkc&}JQcvK8ytsn0L0 z%Jv7s2>l|INZ>pvi5(SM*#ve8;|U&MWX8}?i@0>|lRF&j{VqBKEj~wYW+FkO{Zg_J z84{D-?+D%fBZA5?ja@eF(Xcq~K7i~^#Xe5Z9icw%9&BxoZJb=@tuLQKXpSso+p-UWyEKPsZSMtT$5q?k5iVhQb{42Kw@w8B1#L| z=V*PzT!qx-%|?6AoRF1j&?$r=3NDV)@{S7G2>w@{i^V;j3eMP+o9iJ%*hvylyP+E= z{2ukKyrGNU=Y%@lDAqhsT}CCD1aFRi1wiUy$=sv{W(^_B)t`j+$t91^I6gF{nfe(b zLI&frxkYG50h;vsg?jEFIDtYyi16Eb5SCi~7F}c&jz>fh*f?M37O1#Q&GR~+@;WA; zQqnyfcIBvRi#>4_Q!-`5DhrDHMs@X-@SJLEe=R#HxJ^042Y@T@Z>(Jkz~<+Z#~Ejf zHy#q@W}|5$mC8Fh0vCRD?A902|Lm9URyyjeP^=f+8E5w_KU~KCbS7-W^!nrFK8Fz7 z;DhX|WpflBp?7?QmO`-j;s=zG0TC}#&xF4ZMvgFY2k9$Yu=u0=&>6QylDncvwA>(m zIFOWkLX+V@y(dJ7K?Rj!Jm_`Mzqs75C@%G&Rr30b5J%eYmcnepad53|;zvK!Lm?7K zB2K6lGp^AI3UODv={xM2e)Ae(j6`C(B>0Tzs1BX9a(~TF{1I0oYKLjtGHU>rKf&+o zLm__f1Zw3h;Lp6`7fx^m;SJquhX=WqjoV^r^l72PxXlV)-(qppUhxRGgmer(LBld&1bjEzvh@zccVbH#YL6mOwQ5IOJyDc#+|z9zh$;-Ag zXO!C?D|>%7Wme}TJo;@=ZayrE;%%~YRBood$-=s-m!nj2Yn`RN>&>vBxe&;uQ(4ZS z*V}K%G>x2B6TD(wt=onlQK!rBu!4=4lMToN(80ad>o0%9`2&$Dgw;y?Y7JIjr1}UNH`R{dK5xf63F%q_fj<9TvP&@NC3$= z!Mj%$D8I9HIk&UHL{hf_K%o0psFz2PpV7m4BFPnKlQVv>i0hM=$5yBhw-416>h;xL z5ZV1vCTwu~{PW1S4_MBD-6snqUpYqn8oARDG`=AsJzY+20cuwJQ^D|kydB@(t2!2U zU>funl5|F3-G7a|5*yzH*#$wWaH3ALi-eA<_6ku2=Fos^?}CM8orsq{sifVVfG>_xkrpZN1MHq_kSIYS7!dN36|7A z)7~PisK=t8w;N9uG1-Ah?pslBEKCrtp ziYR_M!-G6g^daS=D;+1a|4sc4ZSc9wgBMKNJo zSWyfW02}-Ret-`9*J7zNrXDts%}3O3mJFVwmnHtshL%L<)w{N{#&2n(@uA2OPm19~ z{NMsY6HS+>!CxNCf9`Oo72PMkfxv&9y_E>+y2#EA^YS%T`zg|hqC$!&O*&~CW>^l+PRf#40q^q-)giIgGc)7iJ6ZuCk!4PohdKj8a>#CieVn-%*W)NN5umsG?65=ROgS_ue`*p6 zhU!8FFqNIF+Tg)WfZ&5foT$$6#C8|s|4mKHH={-l_4>&p^}j`u1JTM& z1s?Ya7&q-$VaWj2Ln+R}?ngIjTPqn%{`*o!iMBH`nv`zYV-~ioAv2C1`W@c{H=<3R zI_y|PFTZA{r^EKPv>2>s;;A^Y9|TwRV1dUk#>Vzm$4Z_IF7^#FDc@pT$=HeQNM-{T zeMtvl_wF0i)15ef*Kk;sk}Xwtz}Ug_dccV`#(rQxI0B5Qg92E;=sVTrkwG*R9O@tZ zN%lHJX$~ah06Imwju!y*b((UmTj2H99ZZXK0myU^>ZI_9 zl%d#RW2qXRRoS11dK{3%++2S08hF^0{w+N1Q?Fglso?hMZqaSfJRa@w>isq>U=g%? zm@xjJZE4GtJAKtU3)D9r?5(FEfFXPfj2x`q5FojcBaB9cdcJz>Y9owZsV^SPOW}TT zd#$i&i5h&JG20nv@?f_;smKK3g+g|ij+crjZUF4Zq|X^3)W`#`FQ38nPN6}(cwx@4 zWO%1rujLfOVf8DAD;6;_QH#9cfxOn@n%d&A9is|aSom59-sqC|PJ|qxuzbdH6Lmvf z!uR!~XDO!MIsU8D-@-YbdXTpz$TtqRW>y`*yQcJ5BR8J|z9(xDQp(=Ur^;C*dn4o;-#m>QGARepT%7>O?48a*+b_#3iY^aHDctFMH8rq)?QwxB$s z9mL2#VSMg{(kplSLshfU@1TQtRsS)Fe2Mwyvy7h7?D{w&lb5W5C&zi?E%n&00c2(Y z;mZ%qMLyH;S}({|>Q=s+?l*Qov*dc?qaN_?L`6<1oeEPPa0pI!OS^sj7%49y_;?W< z4NizPY~W2Aht3POqEDgKe(5UST1E!pRzgy^+MmJpRsn_rQvC=6oL2{35{_+t`Z((- zpk8;{MUG>>33{Fuc2^JDrnq{ZZfQ0RcJQ&x5WZxlww%^*u?bkm@y+yFRY^D`DST>{ zgtg;G7XRf9<)329d7s4O_sztVspre}6?6QfvKvC)n@ex++B2GO)BEt5^+2hPupCd?J})v`g^KB|uueG$oTwm2S4MX6)#6Z*TdBC$qSc*rR>mFZ<{ zxb4UGv8d0teIGc@ljE0VHb0{(~U%P zxw6@V51;W~5D4+w;dYns^8FNqo43A$;zvta zW*7LH>W`6-c{3v^QFq7T#X+}HDAv%|VqbT}=rmuj zD{z1U1|oH4eO6p!SQgIY?Ps}F6*q^|(ff*~4N&8z8D*I`>^s^v`F>V|I0>uzZurh3 z3om{L6$)DWZp|=6F9_PWx5O+)ebkLLZqmAcA>>g?K26>&6(C;>O22zWt+=z0tXzO!`b1r0T4m%lIV^lW;)IqRFBfz4y zA@Z?H>|;AH_!@iv_O-g#e&)M7I@IdKzQ-hujK*~7lPEL0Z=)u|aU6o0exo)ktbYkR zG~Ba#P8EA7+=(1qz zd~A0fb2gMS3M-x++n{d;_8~A9G_{Gi8(_MGO<$Ru?6e+Vb+SjUb5*$6r#?MR8DWw( zPjew>S=gaV9V>bO8L(%zm&lQ1Njy1HVOv>K@sBzomGkHlAX_(dxwM+2etKEqj=AC= zsZDbpFgTjf(!cxthp${&(P;d|YvY*hpL-T(Vle->tS?b6x?Sn_c>k9w5(6dtAEOpf0cY%1W5l`f<~E z*o(Oe3T

60T{U6Tlg?C=PMUPDdFTobF z;tSCfKc;(V5VmS6I8u@i=n8G#x(-R+ki}Rp5l&{;^+k5*oBo`EvZ_v{v29TwAI?U( z#s$pu`B@K}rclRDY;M_Ej&17IVP4pun@sm>=?`YD);K(u3SFzlN18>0euZY6A5$#tCaDZL$C8loX3`RSzCe|3xPtzoF+A zXt!TxW8{E{?h?oYqoA4MkQxljSI=V_S0*TTbwZtjEVet4m|9g>mQ5-C8|#%)TtDFZ z4+;$@B(Zgg-dwW+q(-Bx> zZz|nvhMYbcS(2lUWe4`=I!bCC%6PD;K35hsGL&c4dxJgZ@`$0+uD92SoJ6N+4!T!K zs+x8X8O)xv=wQINecKg^L=4FgWgjnyI?W=sXo)AC|4^?&WIjgX!XQj$`4J;ATL;n0 z)!wD8vE(ko4mVU-KOu|FzLTsv^$h92BQqt)D&DZ`G-Je#yuf{y4a#Q_b<~L@cOlRH zAI9D?Dvl?38%;ux-~?N2ad)=`5H#jN6D~TP9H*g=bAMN%5d`#;aiX zpPSCuIXA=9kGx!mD2kqL4-!6Oy<6OlEwW<)I#4Bw+MqOb^kxTHX_Ur(9(Tm_2pLEZ zgz-ic?QQlEWzD1S`HN9f=*hbLG8r%gKWt&11HfOW`;EYwNHtJj`>&}BF0;pW9Nk)N zN4JfOIIp!DOYxjDdf^t;m}2EkwsF(XMYQwfda4HuPWh(lgyCDVuGaQ*dE;_zxX~vm zk*Ks5Y#N^{34%8>EBmE>9?$_F)=7h8Dq_YY_OlxZ5D!5ObT(O&N3BoS3tRUczbT6y zw{Uj-iZjcLUMX6W<7au75szBi;(Vh?v9`Qcx7 zn+0xw_lI57UZ`DwEsnefyLNOuGAw&e!H8xC2WIpJXnO(q_)`mpw65bmKCzb}SnNLK zdg#O2o{T1ghR+ZZbyk8+a>u_eM_5_QEQVpe7HG}K4al)~aR$2R3VGRXyY51wzKV82 zJA=@Q!5j?1rmPA&<`B6n6Yt_FZtkzzJd^0d{?H3NycB|MP6|D*AS+HF!qjhz6arfE zxS{A&$-4p)t&u*>igICHh{rx~xqYjQQO*GO=fT$d7>B zFRs>E>5`dc`*bRqnJQ$$4jS-Cd&#^$^Aen6A*k3Q#JE(B_2Y@;y0;2*|rnb z>j|<_Y3IK*s^o4EE3$zSpJy!rYhHt=V?I|H+3Xb11T@eKg2QJk*J?AKIhM;Gbv6VqA8IY*# z%?+yrRLi!O%tR$^yEfL$Ko7+>YLY&C+Zs>4$|dE&A-92_g!{*gz!&*oKN<69=$W5t z%_Q}i!&kFROwbiHr=H2s^{vE$g?iUPcZPJQI#=`{&s9Sl;|R)8&*eb1XmEaBI&@$t zt5JV)3ctG=CIGar?!qRiQao&ro>0DrZjmX!(!d#8KWRk|?s<{@vN&@R`R}JaZW2Zp zYE^S^>xbLY0#3sT-biSOr?i4`z39w6g}wjYu)-6N1(T=AMbV)2r?yk!J9c2=%ItTs zx~HPFCyNyZwSCr3M;$zG0kf=onI1cp)73Os9QLyjxF_=CW#afNTekV{*IA_wO8aTq zrkRORSjU9lM?i#k8b0mOYzEMI-TGwGG@ONmA2`LkaFnQf<2;sCa^aomo)&HN=QO{3_vthJZsF%Do2jD55*seVvsSvA0(ZS> z2gPycPjfFUSTu9wLn1!1o4G}!$Sp?gvJhd@#QyhGS|oKs))LT<+R{r_1Dj^*e1Lj5 z@+mJY#U2N&U*$(6(rp(qy7Uo8861svg%R`v;jmSbUvudAqmKl&K9l0RNJpllL?DCv z!67)}<`bkMY2K`zli=(%oWGwqD_DMCGf%=;kCo`=kcjM~j8=-$^??9y*r!|8FyRd? zLh}gx%2p0NokH;K&3uE0IO0?B5s%^2EOmE|paB`7D@OE#^EO9b>EY3m*$Cyg`9A8& zzBDQ~){fF!Sa52zumf09WNNlSzDTgG%qCgsx>$zT$pryxYi2t8yA|+O_7;RxfF&V{ z0Hc1vo#|2h`HdXS>}X9?iOYKHss&j}rfEi6*c&NFdIk!?9H!2G$=l42`(I zNYvl195=9*^~sqlz)esueT@8GxneZJ{=IvqX?>0_knLlzqN`dKEP$|61)b9e7ox>>j0Ry)8)o8 z{MZs%yj{>aMe!%uflI^s%^9mN#g~i2 zxcbCp8e`kqH*wUooo1Qfd<}I_aAG>|99hIP{(|KO^phkT?p0weBjJbI>9G%Zhf4Om z&TEZHmKi4k#Id=7wngT0z0)1Al(gs5hDhVH@9U?+=zZsRWF#6?R;6%(nx@CBfgMMl zAaa_usl#dc%plMV{N6w|yNWuwE&nV{+9PE|I^;K+bMq1OMv=R;Ub?3mivlWw!%u<=csM+3E_Kmi>6BE8OfUYPWzz+sDZ|bES=A~oZ5sbdfAgg<=PDT7 z(dUi;(4-)bAP2Ll#amz%^f}YDa2kY)CL?c_xpz|a$)AnHl3oPijgZIzKZg#}v3m~# z3;Id>?%vA4U$NN~^o8*au-c9&)vKc`mUQGYkxr@^q09-N)W zi1cj?X_{Kyj!8a%`oxKeo)&;aB+sB)r$LVdy=iYl*%2_w-aN9z1~I+4$uQ=c}&H5VO=FxA8+;2j@Ul$N(J@G@kV(9R7w6j`UAhsY$188Chz z5qTN0(vb77M0d(6w~@~ThV=2cs`UyxQxO3-@1BnF+4U}?GGs-^sT^vDA93s-+-S;d ze@}k6$^bek6bhdeg(1C@;yx=<1ufM*z4-Tmd6)hbIzRrL1qxt#J=h~ez0EvhV|w}O z>Uj(FF>Ee7@zG`ECy9{VzL4)E2ROw9*H-RgfLz}ww|E;iQ)~?R4BZFG>_BE_yOwh3 zgAi zPNcxv+~<^UPJR%4T!a?z9G1z{(DX7PnM)-Iu-dZ0XDZ~0RMQ*FLb+M$nv+!rrYv|R7tkbS!-Q*0fI>QOc z;s(RosCORym+g4XKLrUMHN9E1NVh_(OxXz6c*j$%FjO{_Om4Hvce_n^>ld*Y6QnM9 z2q3YDXY~b~iza4Q?3rS;S^YMs6;`#pXhLLhI7ZP@QJSNZg)M=~Sbuw9br}Z%+h9qU zQ5^ZaeDz#j*T1My9_p_f`WYqQaQa4T9xe8ryn9zBg2#5+P)%X{a6ov1Ch`1M>!`o} za6`IRvMV%r#AjMk#?^Xa5?TTOJ90m~r94ygSnAAOB8unPIk}&nz01$4UvIBGv@TOEuwiXdwyO?(SY=^8 zwRTbIIZs+H;@j+5mL5KxKJ>E_8fYC`7@xii zINLvC{zUTR6{~ml+Opab8NOj+#PJU6uwcxfyV07;DvgQ_TTBkNL5zl^e?Ps`l zO0NQfo+hs)m5(%9ohCh;ZO1Zs(4pTsia;%%84wI@PdsjNJ$i4H8gG9YISG5^j8+2B zkLZ{uldw*@rQw#FAyJ=aT9uEv;{8!7y7u3WJrAGq+EtDI@-xD;ftQ4)!eay@&Y?-k z!RcSKTPXnv+cFqiJXw5(W5@L4wSb#7we@$7_n6d6oom2!R0N0eTf;0tOG5HOnY<ngaYl7>D z_=ek_Y2;1Xs1@76-^`N^wF$Q=VO1_vuRlA6S${9`23Gki8vKz5QR^yuA-wr_LRudU zq;u+0=m|kI`-_8YAR3-nLE5#SleP^v`~-IAMS2q;4yC!%p#u5|1}u_Dd6IfAn=tUmuB>R7)WboQUMt1qMWGP1Qm}x-@mpla86ge$9C|dBb23 z1r+s2XUWX_!-WhEV;0Fg^J=NY?wBR~m`P9as7T>?`GB%8fV_~HRU$QUyO>h=*pD^@ z#-p^Lq`CBMlF_Z#5aF1-sfatg>r5ga76Cryn7pgNy*_vrEk$f3dS6(D*c$$LXnSze z1Fu*q3{15|mqscdV7T9RPsW;?dKobcX9@i-9fw5;?i&XD?NV8o!}r>`)>*tZmMMnE(j)}Hc1j%P;#5JW7NMVstio^A*8P7)k*I};Q~yq0&Hl~ zq{mE5wQ%TIiueQT_6G-(mY6NEmtrz$n#tM2Xe|>c{bf;5++XR8&omH^2O1dp@?{7` zD51#*F;qXi_xUVqI5bZC_Q`H71sc#^v4u zCbjdrIcYEPQCL1;T0t6^sR!SNqJyXf0J&St0zjNlH;wEWm-{?Qydxm?rO}%PVO&-l z{@MS@_ffX*CFjBCK{iNT>atCkR)UW?NPesCH;=P(_WS$(uPK7NkJbU4BGlxp*}^iS zrQz)r>MXxwH^aC9snO~d-H8o&7qozg8~y<{kGr!xh_NmVOXot%1O@p?aV{=Z<1> zf4m%aZ@p&!kq3`Zr!)wbuW7cNBwSUA1!iptV4JD`WmNe&c-5`yTBHAU6DanmG z+nd5@0R-hqw>gWoiqUFuSjRv;$ACspG0Hl2iuwG$%(rP~daw%wt}o z32r<<8Mo!wqD@3kA;1JtSq$mvOHK5U?Q&I0+0cGvjVD5B!}^swULC@vSFbXBmLjHw zg6mjM=6wyK2PFo|JVcm%Qc{z_92&cquPVRn$$|=WS`yA@jb!3m5k704Am#`n1o{jbm9i8yiT!m)dNO0-(a8 zb{gGc4u)S`CR$B0s{&TEDE^HHxl|AnLzIUny$PXYq0#NS4Swa}#)%3LVJtPfuhF8h{Ns3^=VD zfvW?`Uj__KI}`rRXr_JP&cyYD4g{3QT6_zeQyQZoF=jE^ZBw=Y!vlVex`>RUKHeC& zw^OL`ZG2#1mrYayzw?7#kT@B<&uS_!1?CO@@OyD}IIH>?S_M!Plyvn6U+-y#g4ruH zkB}!nL|euskiQm+{Rtf;3@Xvj$htr|T`4Xj9y{ zrm--IvfMCkZzddJNtA9LG=&L^;bl@;B3(jTnM);SN?L-^zGdnX&D`#xCHqI>9=1!5J^)GoLUrHLDI7?N21Rx4opORgNB~B zEe>`-zJ2jp1EcQsD&=PYxs=m`1=3}|;N?@hzi!I>P32jT-$>ygb1UgOcZm#+B>Hl4D-LO8m=B#K`s5>%*2M-ea z`&li5Hrug?HnVUH68tAlR@jq=aMegMyH&_wysmV%u^n9)0QP)cl?e%5ZgqQMnQ9^06R&ga_?(?%|z?&d_UE&@pmXmoDgYT zh9YN=N%(dJ8^ptvfcV^B3p}7oriM;1TCJdr4Ha6dp>HsrNLX^LXDPAT4`;w`i+>Tpzdfz^P8z4BA2t`8uN(`(zOQWG$QcOrfco)PCKSb zXlM-McNi+>d7Twijz6`TsyAv&DKkgz8R~nwfmMdNxbE45UC_J0@L%39L~dY1fh<{? z@nVRQavmd#LQP08-fNL(A-IJ(>6zty&0cVM|M&3<7HU*L*)rVK?)MLCAJy<2Y48$G zz@1*2J8pzNM#{&#$d=V0?%bOvh;F3)BNbstTY^@WxGJkP{BtxER8=(a2%LXHc7IiS%ni}bj^amXLlFPH-`qCIw0@Z_PmTCg~Z;{*mb&FloJBi zhb?ECU?JME!-u>N-NNaXTt<_S{iv_gW+xu)o&3OPZdWuaJ~BnR@)~J|DtDJGoFWoW zpC`?OT|MS$x4dNjQRS1>IE)_Kcm32)b$y9d#nTb|Ptph8eP{epFA#*M%y7U1zU-Ef}vebcjDo_34E>_<|}^T+SkpquGR+00JXZxWUx>Lw+) zI&h|D7dz72L6yr&RFqJ4G(U$XNHubh{~@I6+eoe~7qRE{9D1I5kH(MWJaw99^Q<93 zSgHkwx7P-d3jk=%xN~c+QLYzO9d*8$z@fq-ZqjQj2MCiYMcdaLf%P$8<@0AzjxxL- z9PIu!aOf66td)jo(3!UXacJIxZVTx;aTmBAVCu&A_s^aQcAV4JO4g;~tT)Jo&%|b^ zmSs4OW$EDC$LiPp zpNswMe@`}lcmMo-Z4!H(eSNzBN5{h$a8vepHQc8A>*ySTzjH<)_MaYtJg35d{>rOr zu-~46WzPD)yXEY^j(@!W-ZuUDx3|uH`SwrL&867SXY#T88!>qFLowgy;dP#i1U^T? zqXwVT_l&G=C@rOG_iD%z;gt^8O zkoN!XAHa``t=^#u9n-&knCODVeY7&_h5$`8T)zZRJB`fNAgjm?{$yDL(xYz?Zv zJ-!$H|8OCLG(DB1Hb+#!l)yPtrH%>bUM$tB6!pWs85_ZucC5;mdPKz3sq)!(FnQXpGiBM$DJE9JchSblL4yx8e7l=aqj z7=#C~@k(5&UR zNdojG5Zc3X`H~9bnOZ%YNgM$UxY8s7l{#R2N@G0gBZX^&1*-4J==wFIpx?KYs|vRU z3pjMSYHmK!f8uMBbgDnNUGeoMAx!=dq%tC)391l+TxP*0tW#S%-Yd&^iI@hbI|n8} zLj#$iaRL?s=W=2| zEsxubMV;%|W?*l{2&{OmKC6OLJ1JgmN*!kl&MuWwcIHv9B=`|D3!M7G)nxyL2L!>n zqmj%{P~G{r!LcK81FW#3{YXy!8XCNu5lh;T)-G5j+4ogQExZfi(rQvSEaA`*omNvm z4Ry4m|LK>E$ly00Dg&_xX~QzM3wET&wGDN{GV9tp#KzJ_hsG}TSwJ&lqbtb6@Hl7^ zK}T(xNC*Bnyc4mJEv3s6xUXZ~-@Zd6+kF!3@|`!6^PP?=|2oH(w?qJS7SFZIulEfN zlmSdw5`Bwq-60!4!;84iJaywvXqPmD7yx(a=k_82A8S|2NLa347G9pn!6bm|);{tB3#f}&OD22l_EP#_|x%ja=!19gVFhUUdFQVKETgN( zeZy#}r{!n;a#}Jd4IBYORc-a;$rf58+kd}>l_rIU5=)irPkTlEjR(OH9K3&OKr6CESLn} z);pzKwiEEyH$Nv1Q|t1ux~iq19h#5Z;wW(u+`PwmODa^W=eU=a9Mokg%DBfZ6GPAK z>5=UZI`vZ_kC?9{PDsr-<`K6gM8DSrWqL+J2R?nJ?u^Qjn{VFRJW+<-1U*7vF-B=# zoc-3O3ir7S_JBs!gZ1^#_&zKEC^uk=cQHfyf-@lkKej@^gj;WU1q$mV6L*6-g5L6F z@s~hZ(TQS^iE)sqoc9p3hCVxs8Su84>~ZxVX#CT4wQoosrkH4vpts2Cn-IZkYCJaM zDCvf@>4~cE5eTBwAyI3hpn0*|b2XT{(_x;-fy8kulEo)e4hhT-LyM<*_)i zsko#Y6ng(Vr;sAn?$lRsp>HbP*gJEbtv8QKL#Uh3yd~*RTSCPzo5*{uWe zdQHIW%@J=!$r%3h5Py5O{_D))V4%V*PL#{}8WZ-4eU2s=*(~|TsJXBJmI12Xzyf9P zng&Q>cW!U$zp1JEZb@dsGNr&qYS~laY+dE;AA)9bWhZysKk|EhOXFyhF9Bt3y4;c9 z4qb{*7I>fA_jNShI+{LSYopua9+jHFmZPW%-+T58W@Y&gxXg&sUF;m=))VSLNqvTp zrDV;R*EoIG^V0%R$V~XcdnVgdRW9pbEE4E8BZbI;+`PAbJFuPN#zwT$f_Sf0IR5a7aOOXKo!<~N}1 zE?V@+HP2RT$MSy1HdBSYq|C9}z_U6|0xhO#-MM6DbM4Alh(&b~_9!*7vO%RRpREyi zhxLRT+!fw5cCp#93io1q#4d9RO*0qWxz4D6^(o}{+aE(W=#SAt`> zS}QB@nK}r*$MzEI@nKa>S#^Bl#j29Slyi8sVtM`8BNpr6 zCA(<>qSZPaqz=?(-Yk1TuCv8KEj``*#8IUW@Im_ zAnP5jrf~JZgPfy`#r?mKZZ29!+Yt*z7Q|=D|HA)*l4j?A882aT+wC9`UZCAJmbI$m zhy*0LIF(9+4bts7K!hUA_xAjTPd}OndF^>ogU&)LcrFdcv|bO-%IW_1FIgmm=lc6N_LkgjyAk-k z*apzWg#3Q%%O97)N7tpmoj=3G3#JZh8*=O~C>}?YfmE?LcRLId3+k_wDnF%BnhwJy zeXcU#R;Ajb@1ooD?3kyD<(Wp)+l`T69mg@|#VYFpu}XJIs=k_qtP?mrq6AWPN@pfO zjU0&Hi3uh*Xm*aJ;iDCKTWo9FCXPm#{ck8S^XDw-=l9dKUDQ&wp zH(rG5OC_RsQ#UdMJ=pTH}r~@V!TgKw?4o~*MrfTH{S^)?H%D1FgI;!6k*|qD6 zaNQx4UK5HvnpGK_Wsjy68f-g+Srk-9;&;%9Y~o=luU;2Gr!MOU5ZBP8I_X?a@Xb(h z$=t1+lW(9AYYJ$Z+f6Z~U2cD*BM^0kC}4KJ{XOxZu!{)VPFbTW=XEps7R4>8y!z3{ zLF!m1*maE8;2};;=tqmMV1!L{nud>cii)Sq3@lM%uf|WH4}T2BSbx6jJdYN_6Jc^{ z9L;Su3`qPbjK>e${F+mZ+*D680>iM~;lQE{;*vn;7a<>te&u*&yQtE+h#wWR?l$-i zpnE~Qw(VYTpHu{QVv;L7I9!^iUc?KE*}~ROJT0RofNP8eH`4qJG4}VK^iN6?{ml1*JuV#& z?LusE&P@kjLWraPijzbzH2iJ=yawLzDvDN#O2O5dD4i7!nLx0|>vz&x$+-$$gG{mJ z@{Guze4rKUF(cl;lk+06YqIFl5o(Uy(7R;WUUX*I=j6v3%kq&sihHIdGn`{k01+SS zSGC8ku=~qPL!pX?p*u`=YN`%SRn(i7d}BsufG7b=ajbjJH4&{mYrXL)RADW>sxsZB z$nC!OyO`;2%!j3{u|f>G3fs2`%8_Me(NLiF-H73Ppyv8+y-yM>Co8c<%nJLs{N<+z zx=eLuJF{p^mcOR6S9S)d^GH^c`2BSdPO@+apBWO&r1~wrBlqNl7VnuIKQ=HYN|r;} zH#nBjHL-`}iE^qh!+` zXD@LUqwc=eRB>I|F<{2WoM>W!x`U+O8Dp-}5Ve$uWKU6l#0~|1N6=_wo*!GL!Ue5fz8Qqj`wZ^(s5Z@@)dR1Z~tnQnJSPymgO&oRr17~EXYg~AYgqY@)1l9|DjyK zBWK^=Zu*NnSiNpOFAfkTDecn)6Sk!GFJvwj{-Uyfomo&^^$bt0={>L1{W}y@40hiw z7v^Pt);5nBBj;JPdYlo>J7C;Z+p_JVP=u@UYhreUhDzLK-wlfeXO!Ri@2<=7njs|p zH07whG}9Kr#_;me6v1}$s-+%xN?<-TG1EokbGLW7CbSAmEg=W8-#1%w@#1Abv>r`j zB!aY^inMf!ogb>ApAKon(LV zd9F5^rFTa`m4&~cqt~z`a>a?(#omLHy=Pj9FZ^#*B^nPGLag_>Es(*o_E3;G3qa^- zq^2mv|L%dBw6dy4_FMV>Bm2j{A6FKxUkL4}Wx3Lt>;kyhekzgp>Ez0Oi^xq!E#0^) z3Q+xg8UBHW%lrp5?(;x}DF0PjvJ?JSV4!w%AQ3?q+85Pt;9QoN1; z5blln;NayQRb$N;x>9ZIj|JX z+Bri!W$B5Hupp*!C5tc460nY>Sp9{xFG^^`Hj9O{VP!8j+7W8!KAx;XgO((m9RxnJ zll;bBtU?~|P_KjLjU(_^ZnXJ$R%$*!;v?Yv#fhBJtDo~05Q7m<5gH4K<%`=ua4Kkr ztOe{Hk?(=|;_~abZ|9DSCzdWkJP@|Rob17p4xzPMC4DRDIz(7-%gL~^N>PkHYd7}3 zkGmRx3h|Y(hPD7m={t$a??p5qq#5AZW+RjwyT-H1i=gb)W`opQ z{FZd^?eYksg2Aj|pJwqY$*ds~@+NTB#l>kH+sYxaM#l9Aj7(qP6k*H^&(i1DXL|l9 znP!{Ay)tjM2H$6>Fh3tfDh$(AnpRpFHcyDZ{;$hCw5j9uK zdei)jY|4dgjh8V$f*96J`r{5zf+ax?K!lb@wf*16@;5apQgTl<0ipzTvR^DHza^W; z#&yPqOW+VfjD+0qEICu6MJT`OP%IcOFRO@0*ZQAHFLtR6PRP7tewKDDZ{2+1VM4nI zP2_q*^gqaT3d-T5Byy(~ZniL`KX*QhUmOJ_}-ArXwDT zuHjHg3UYXj9#N6$i``l#SHg!umI3VKKG;7(C zA-oVt+E8u{`|mb~0b^M!^iM{DAmJ;$3zq}cO~wex9w)tvNp@O1++diN{`yCGs$gn7 zzgT))i<1nz0d+liQdsrh%hp=6d`a5szvYkg)PFnNm6gk_3wUTH!a1)}>>g3|;ZoN` zr(`uE4h^luO7%1N0HU?k&B}Y!)Xlnd&D7vdMQfS#?%^epL2Io<(HS*$+QtnccYv8% zE6ci>nhuZO%@cOlNxb&vOEm`f* zAKu#`+-SVCC$}CEKcg%Ee_!&Y&BA*MT!lAWyk@5Mlm<_bM&YCC)Q1nZQ~$q`F1%Eu z3ordo!dX#PK9sW$pVo0Ayjy=Q_^Wj7L;t%U8((&}`&!@+;RBovf)kKy&o@!+L+FCx|6PQ|^I9gKI{1nk*E#Gx zwv}7wYyDTvEm6(Rqy$a}5%?xTg720?8!!00kKrp#bOWc_Q~OyhQ>GaF;eX9e&A-Bz zq7P1*n+_LvHLjoVVSs1g6byzhYNa>4F4;JIJ*y7k2^_xhzv{Y*!n=I}whxy^wEVZ8 zznkH6Y4~r`fZ;0$5rMzzx6f3zbm9HG{7A=Qmuuem`Lj-iRD%`XS{&#N7%=;XdzZ1r#VqgPil69VUy=If0|+O#VEb=PS}Dsl@w+mRf8sp>-$m@ z{>KPH*;jCm9^swUepdQ}hY?!;?Zh)&y4Ru~L3_Pk3y@!EIj|VLKq0|M{kyFKXM=X; zpDjp$e@$l?=#0XGmuHHRHc%y86=bl^rWCwTQKhEhM;tiZI_jz?M}#xyLF-x*Qq)w{ zf@>f620mv_f-RsF$aXc?B^W(3!g;T~GsD!g_9^GB#f6hsO9|DI9|@GO7tuSaEIe;< zDI&RIait|Ua@nO_^$2GsIWo65`xqm3RZtksE5)YuI?E1{5Wn^P^PNCp)D91NM)HE- zlsuf19FRz#!>BI~kWE2BuTO8ak}~fg7CTabPSW}u;})iq+U&L#H2dqp+*dm7S{6H| zV_4+zLO;$-uChqPO`o_xwX%Gb-A{IBFDzt>UC<1II6pz+OG}hEzy2CmSO|$84FbJ9 zb&e$8h)Id%DpzCvanPpLamW@2fank!i>o*&D~kjiM>!;wEa=}=FHZ!-W;uD5*9lTG zyMmnlKrd>r4~;metlu4`%9My!HQT4${$3bk%iOu?H2T}>42-7sm491Nhc^D^-_GRk zq<8i*$!j!QqOIsD9#`L0hp#JlN-!eRK=7a3pehNv$GlbeSOJ>J5u(Eym3DpX%OF_= z@}q&6UBCa|1EU`}Mx>1%Itl$S_467h@p)qne5AU zaZu)U>FKv5?>m2?^{>Ur@MYqnOJ~qxKB_Uqf6Q2m(k31brXO53Tx(G!0XmUZtuXp% zPcNS=k}~CGzF9tMl_^!{#&U4M^RnmI>ggL?hxG1UiV{LRy|crkX`@c~!sQ}o#bc#K zo3dF>@~mz(749hhl#)J6^#H!k_f9&_V<@u14Tr1bycz8EH|3)OHu)OyzI-xC#^5@lUN}fb-S^gw_V+X>Yi|n-?p8%%|7Ng8idKPf zWn~g#L&ei^P_z|0FI=Ak$L0I)1=uhyo_Jt6G}z3|LEl(H(k4De|B50;-re|Nj`OE-7RbBlBV=>-C-PYo$J-@fG0 zfWz`>iS7Rij0Dr%$938aD{h0PonG=&WkulOV6I5Z-P#)7hVT@0a`D>(7D+g!zELt& zZ{m!;waUT4hxR&io3?$IG*(laizX2v3D9VJeit58fV|Z;LOy`xWw`NC zdpw(G%z^3M`1}W_-5-bHVU;f2>%;zs!(VaH95fFu+tAYpp0*~&(+>D4V>1;!l&1Ok zRoY_4GAS{RDcplNVA!JocFkjiRVOx+a^|t$@;7#Hr-|U|a*Q(R~?<|bv&_%iUr#T1L`3G}v zp81>Bfyrd!n*7`xKX z0KK*Twx)dc!pe--W*La47ryT`s{VxPLLBz_f+ z-Zl3SX&sU}IO^_z%FaD(XOn|6apKmDn%nS(>L2+4u>l0DH$m=z*n;^&{(6{W^oNR* z4UE$pP(!T=N1n2tblv%sqSwSQCU6=P_IvO3`8PN$aeD7?(BE3c* z+G1n{U0B6@o>2E(S{@>?v^}5A;~*^UxRQ4z`LWo{yaCvu zUiqOdpE*C?2Gy8H1WI4o_h!nYYBuUV#^}jL!Wjq1>Rku+4=!3ao;tKb!w7wMOIQ1Yh{fTtY!BF1;LNt0ZQP;QOG9)W(S9n7sEA25zT2_wa9c0x$j?*TzruAY~_BNYx zw2oW=cD;ZFqOm#Cs5;LK&buI`3^24Vp2TXe$sE~!%BO4=b89~c;}=KaQ@HvV?Kl>L zk>*cZ#=zu}ZMf0M4e&hovn6t{-rvkII`%wZ}Qn!TX0@)B{C=U z(Q0m>4QC}il}`S1ud3~$Hk<1++~3->19s$IS}1ZJ)CfH9sO!O*pE8cxq_^`Ub$ZCL z2_}f=vF^C6c3q+cr$?Gkn+rXf22N!A;4??jnx!^r5I?yjwIHTcqt^ zQiDjurf68>zUb>-(F=ChQ0nx%#S16CuoT#9N_712Imf%k*X!u?z#hw3lOF2D?0$d! zIyR|PeK!`%S#lYqG8Bw*mHzkF+UfMQiWK&vex~WHwi1`l7u;ZbR*SAU<+j(L&6}>W z&q)wof;n6*#6DxhDIDsV=5ak5=L+O77N9%7L?ppap_84B3On4)>{R5ww5Y8-tB%rI zkT*d9H3lUqJKm#n6fIi@o6S=Yj?~SgU2~at)WN!bwAp^uLL{+QkTjj*_GHci3!h8~ z(v%r1$PyCxq!%uN?|Gp>9x`&oMV&OK*=%~lsR@>jW=6U`4EeIJ^gM+VPwNUuFBTR+ z(jXad(hF}>h0D_a5*{{|jnX{h1(BoJzQPrH7^eI5cW_T@UnAIg`rC?jKJ|+J=hE%b zQ+!Kj@Bd-!t)t=!f<55`2<{TxAq2O3w7TgK$8eD?AOTNk5 z-S^&}-S3?Jr|(SNmg=s$-M8yj{aRmnxtQ;Z|BwFxbe1!EB?yItONakEZ;%1$o2oVl zpb_oCD6Z0HSN~d>Q5IHlA(y9m;GI12#RvvhZ z8#uZt;5{b4uk}>R`f+d>8pWk?cCC*U1Kpbg1-eMXXH{RB!l4t1kjk^&0TkN4^m3BT zaE0gBUiTWkair0!w7-<#619ubaFQ0#d6waWu?&;AdPLri31OLD{|!l+>t1JfcA!CT zvXEKq>pRz#+GdHX(H6>z+caDZ?fmBDkgs;!6269^C1ONFw?>ZpU#!~LcVeHksBc-a zOrxX(+qCGlb4e_(VkBK{I3EXy7^!^{Wx_mrrY1vi#R>aoEX97IUr#x5vZWRKUQAC} zMqrJsJ8Piny3+L^zNrz^364h9?uwJ2`guVnNE0W$ydW(2W${v~iXpq9_95${;bam{ zbIt^br>B-p&6MHvx9O+$)AIS#lj^DG6bsAm=Cs@jDt$@H0PHEAD52pD6N1)QM5LsV zOi8+#LSo?Lk9pIHvL=-pto770@H@-5_`Su)Vv>b)KLMO(sPDxe=2l50ZUA$xM^yaM zFdF+Ev1J$+KTMoEptzan#Ex`?KDgRD(6@v=v4w}WpUQ;@+B?~OkyT#+8UkRlWxV0> z?}Y-f^-5j7*+PbjUO->Amx>QZuj&#b*PmHXKp|MQRm~r~`%30-1Y(Eb5sw2`jau-& zukmkkdkZ%io(Mc>R@RK{lXLYsP)TG_q#HvNokFnOf|p4vupH~SYSqY(;pARE*@k~; z(h~nf;6U!*kGQHueOA(U(ry)rUiOR}vBeFx9i73;uU|Gc*KP}@&m;{$Gmu}CbTibw zo8|td4sg32v_9nt~)&{;mo@4PSQ(uYO;Dvu$TRw20BJYGGLa&EA4Sp{!(0piqQl zCx%A(Kx5XXvAHG?aGA(V4mbcNs^M-@W)r$4w4o&DI%jhGwrwk8yGyEj&0K~TLj0Z2paJdO;Pbp$%PbExgw@bKA zC*FyyiG>&e$jSLY@IAB>EMEm&`yue3`O3y=ij(@VT>yvHPQW92qxR3+F1bw|w@Pb# z!F||vcirM5-x;pR9{c?Gxs=T2D2_!abvM$~?`Vpcvjnm%OL9*p7hP#3`qYRR>zNtK z9G=VhHkh1;d}26d>?G5eirvk5>rl3Hw(uN!f(`vioq0jj9%| z~!Iu^uMR9 z($n`!2*fD*dKAsJ+3D44A;V}d8nh#C)@3ZAhxHvj!pmxy}*M)D>d5~ucp?lA5eWltnZuP`0H{QhkW{L@-`Mqql6_;EJWxW z3(1^qdvBNHIhY_nWY+)Y1G@y>taqIIoq%J@ExV#iQC2@Y>0pTL;9TNgJ~Ad@MMui*?X8 zF7P<&jrEXsDgV8Vp`&@{YzxIWyKY`|ogWyW0}mMW_e8}^F8!3@bP(2MyR zf|q~w`)#pF>BnYp3Y}yToIvziu2Ok^P|#c_u%n359w+kY!F!ivY6t4;=Xc7*abnL& zgk)s2-F5d`%vhs@ow@g)vV%RwOgNTRtNLVzRJTy}Rr`fIL|o1iuABb+g~`?MzWTqo z!gBsUy25gB0yvO`g(+M?F6PGe$euqc{Z7m5TjPHJ^^U%V=~&qy$Ar=6QGXT4qrkv) zAtvHaIJ=Kns1aIhD5S6KNFEPzRT?ihA8lg7(5~AJ;Cs5p$bKsmdU-O0I;{SEMKg@X z*y-!~d%p2@Q2jR~zB&Wp6nbhvzX8dgSJszD^1m;(Z#x@*_nHG&hY zQw`Vmf{WgSR|ZpwEO!DHhg~YneI(lmj4H(| za^U;Bg5uDimTrE6$`P!8mhL*Q8j()H*gr1?7fu0B=Qjt+VRP>W*#Dm0ehUb@`EXhX z8aWSmy^9ZMGZg;&bbQ|RC}-)FTJpRe@Otam^@@L>@>t#V_Oy8V_wMZ*Mz9E;{Us~?TGN_~0KD0%VpzB;$a564zw%=r=$d9CVr5|%@KVSeAo6`+AV%+-sQg(< zcbefAodjEdNW`!NbpaC%Q%zpP3DXEtYX|fc24lRrbR=4TSBv2C#j*Yeoa-5MbQqk< zDae{vfJ+90J6UGipEbr?or>Rt`j9H``h8ySqp}Z2}GdgFh`m`n_Eo*-AZxit_H~>i6kT#8)%Bbj?DT)i;&&c~TL;+=# zt~#fuTBv3XX&3#I;D&9xo#v(q=^>PevROaR;tUX z;Y4YHi$ZCWYMhcHvy*)GWTk%Qz2SsLOh+arUxeU!vuSpTKP4}wA7?1*|cdk4<*TkB+Bh+j?pE@NjHWF}3;fkk%e zlp#Mv21)#rFRR{@DDRxl8f1nH?#SNp7DE@ya-vZ^u9IwXcBX+oCi3c$d?)>`HsiYo zXsn<{P+tt#vgp&HuV2{L_QtajoXJAkn4Db!(TFuBe=CHpwwP%mpJ0-^de--B6cMll ze^#kP2Pa5B=b}oS&|nGS{gtK60kjh)Wlvc34^s*W0pwIvne$oSn-f)fmkyBdrHicJ zUTZxh&rYE(JQYxG0G6khkcvtDkcf0{H56#SJC9&_*L|mocwxw~3RERPOP!PFrByG& zVkoA}-BvIzLVG%j4QI73;v6YnsNDPY0dqW4mPzxTp_P;=V!}FdZxcc*Omo7TGdVsX zN$ru5J;5#Dh~P=rV8SfvyoEE5EZ+r=@%lF35}9Jlvy_gw)cy-kwXTOx4{lkE2$sb_krK~| zwn+2v(E5po=!?|GSS{UA%mUutBm1`Vey(oRLi3%CT`ezmkI+|V zcTKFwA`PhOCcCOQNk|`X1zo$o^>blf%rB!iJHdaCfyF;O-OR47vmd*}Gva7l@gZH?_qe*96$A?@|+xG0`*?E_#K!^gl~cw>uPg(Msr?3w?==< zOQODCTT;$u!oqTNSZ7nt3R#eH_<0$8GJBe6M;#9JdK7I3O(J)p6>`?5ct^`00Gz7Q zX<8^Chl1QGfn){C91yIU7=W&vc=0vi`X=`6USD#HJ>a%2_U9^A0z>sEub)=m>q7mq zI#qxA@|%V}NY`-f+lPBydW6w!Z05ekg%G~+1}`BytxV-Z^YPa~aQJN=I0GzYpfvQ5K+;4{Mt7Cr~>$t&b;!zxf$Tfk2A;8pv=!tl$3a&m2 z9dXQRsU><8^umS1j;FLawn|*H>Cl)>oWM92j9Ayry_xePeDE8jF&>4|WkZQb_(n{t z*eOFqcnxL$9Om0!O=*r`GbDtIIMgGMvQ%7^^?RZ56U9FKe~=t=)D&VS$7Q& z_{(8KpnbOpgcJtvy7QoDt2Y?4+iRn~k(%Vy?e{&w%hU+lbW{(9{TiW!>4oZ*7^<#7 z*6d9f0XGrZy3{8f<5RPBMb|pk3C^Bqd4$FhR}7piyYZSsqzP6~U2Pv2W^qxw8iO%` zQY*sJ0kUOm*ZI}()GFe5`OATn(W&QIEV`Gz8{tc6lVB=T-YR+55I0q1@%%%2_DrT^ zGD1-xaTHs6uLK`4(m4_bFU$N57ai>5H=y0r0(v}Mh4}ymAtb-g^Z~osqGd$fa_Hv# za68U+SMVhis0r$vbG~+;g*zMf@S~*kBbMSfR6+c$g3toY_Hca482V!flBcy5HNFn@ zE)(+t%L^78HH&vQqHYvbxtrb{1IGw7ouTQUm_S>G_5Jj~;~QNfKT+SZ=EdOt;?x-? zWl6PnmE}B-l=b}cDop;^#*6!_)A*XHQFi;#X-JdHgy@xJ{J;n(UVY z8iwlA8O+?Wj7;=G{}gCRhmS#UJx1ofwf|JpZ~} zJDVxVqpd^*M?UXgQMr%pzzt4&x)@)IGnCc2dsC_9C4v6 z6wO>VBLpg3PCG>4%GpnW9as;p-Dny!sy<*Yfgv+SiS&bL@o zs}_dSgDcNhn;SG#KQcUjkiBFAwXoI=Kk82pMqfA_JWa>du!*-Zwe{|s)<5-{PanR# zYr;5xf$B+!qP5orPj+iPVy>^}1zL8{uLy*VO+t;0i!w()XU3F?0p4I@u=~nt=*(hy zADQ(CiH{@WJC$=rUM-|5t!olNrX=?(YO8m@_3H-7tbV7?=T}KaI_WJ~j1Y`961G`2 z!0OtCk{n%g#Bq09|NWI-M`r-Wi5sRh7x92$%DEOXQu;0oof&20Y^m75i{!;M0W5z~U z0Me%~9NLP28@>F-j!lE3j&xOCCT2zIiqAyeE=AGNV6Airsg|nO`*3<> zdKa*^`X&^b6v3W zbRN$}A(7x>-FhL>X~N5(>%8P=${z-`%c@IA2d%#Hd4a!Zgra6^=t&H!&3IrVK2kqs zB`2V|<5vb@?9hdqfRQH{&cj(cSGU3?yq}pJh#Z!>zht~&xH+Q1j^y}=p^^?_6hDR# zi%2%~Z3Il$%19lwHW$dnXxJLCtO;_84p3{X$NA2D83v^3ONxoRwV9q(h5AYK5xxIK z46cdaMmFJI%A+`2i4>{}9sPu>ia*q9RtmwrmlgpVs1 z3Lyo4RQ7^?M?Z%X9anQrfARMSTp&*Z>{18!+ z*&_BP4!1EJ=p#&yp}mT<&;{nKE4If9yM5o9J*2*oPkl&Q2j(ysXR013^f@x2obK%Z zK*|Zo$cr$r8lB%BJpE z9>=UA64pY4$W(ZH*S)!E-FS6?zG>4?k3SMzl2;xH(+{d_;V5C#4}UFe)24EEr2}BX zUkdk-Pi|jl1$>qW9bGOBY-w6Zgx52H52z?A7VmS`07^(KD8*aR#Rf8bVE2Ufq}dDG z1b^DSHLK8ot6$}f(@p`oT~{{t*VMl8TQiE@hM@(=7p#Y-7}C-@?IY-<-PXG?t=6H$ ztc$mB=C)GO0=1mkb#`(L73adRytm&1jk5@mnma};EGn6dH>8P5VLZliP%ly8#^Mfl z=tUFc121&>25%DNb1`;gax$gU7)^XY(xM-(M%5W(~0MED_*a|H^(^PW7wnmT(=IWD%Zn~b@uyuVUP3-5jZ3$QCaEf?ut|MCaf{0png3{X+LHJ>Eu{F=@s?>eFr$k zl{afLv1t4C2v6o^|B_l-m%s^WmDx%M#R6$aDZ(G9PQI?2aeRg8yfctgMZ`Q#{-_E>;<1jqk`n#_-OS`O8{nh7sx~3b0|^k24HF&i`CgU`RDf8cUrDkg?5!y zzeEi(c^olVIHHl3cxhK02_yw&sB%1ubw*C5;YK&Bns=Zv*nDvfHH&SYgP&xtY@XAF zZU;Z{x2^?h9j(?OUJ^Q%f;**sWv>RX#c4v)p#N%j;7A<3V{0{o>vrrDrWIYK-NIsvlxB=TOe!E zYIEkWEailyCGkRqS7qEhAapp#xKd1P!YGBNdlZKTol4XKdu!V zOHGoYZDv)MH(Yb{UaJuS2o~>mHK6ptLefY*B5#vPG(DT^eWmVpm|7{P%U46>hjNSE zO?MTicd{nz)CjkI~(7dj0~0{aa1b_KXWL#$R>}owwh+Nm3m;8 z_L$7A&xe-?Ps~N@v6VzKL#1X%u$7{rggi#Qrja9&Rx>$BqPe4L-FwSZE|xkmH>|WZ z>Nzn>C{p*PK_9+cxwXQ$h7Xe9a#T>oL87A(1Z#3n(>@$b>Wtt$Z!u-b=l#xPk?Bx0 zyk>WYzVruaUHqbc+4`a%f;UFY<1Z{dKs^${>88JHzQrpV!WqwyE0I3(KWf2$%t-krn1VEoyR9_-x17u4BSuoKRiB@mJG=DABIHB(SoEy6oa++U6$%Fv==`+g};)JzS!|gd)aV1sWZFGwdz%Ez90YuE#g>S@UrMZ4LpmalRZ%R2u4t-`slwS zKBu%|ok{`+4&oQDOGhG`Vu>xfvu z#u-uG>*8q@83=tkZCd#V1$lE{7?OBXg(>j0QXyzKD2!#i^TZ6oCPyEVyzU*euI*N%KQtroSW1?mmPX{PaRLL%{#(2a}I0 zY6_^+6E)P%;m(NAf#W<*W+~^qc<9kBHmv9*5X7vofee`!b}X>*Ajry zf*t3|8K*Dk>vtwfBxEqa@Aeibz~nF9g2yAp&8rECq5euUKv<0hl0*nxuP^`pu*uPQ zU>EJ0N4q!FEzYtE$p7nxs?3oOqgPjeF#R_OiD0O$VEb66j)~>f;$BUh+?dJ9TD`AO zNQFI~KY1QeR*Dwv-PCt!)+#%qL7nl!K&v}nW3r(^zYGrT+jakz#I+>)-5f}rXKTQd zp&mcx?ze8`ppOKSi*(%$o0sD1eFx;Gf-usUkVP4nV>xAObb5v4FsgMM9gs5AOgkmW zw^@7&+E~J3dE>>Yb)bQ1ZcC4xC-79rr%Na0ssdfNc~XA&h1(?3f4>~}N5}N5U)a;p zOjxK8xi=AhZv~yU_jdH{ye1BxAAI=2frSK8EFzL$=7z`MlYh&3EDzDv@2+hNa2{-T zi|gz#Vj!c#0s%h@#*myjgmq0Q9)A!h(BebJ`i;k5Vm;@WY%wjz|Mw2j%e$2}+=^SG zDef$9son*~88Lh{38WjW+S=*JGOR_(s%dm8<-xF;DpzIg{CJgbPZ{}k60w(SdQ^Qj z>AWjchsWdBgK?k#xJhmv$pr-)R4^hzp5flwRMMMg#|5R&9(46F&~HYqA4)JyX3~WB z8ik!BUh}d}sxGjy4zM7^X=0cjODz~xMcq5uKA*rdA7BeVVEc}z6dbT&VkbeE70-qjww zq{1E0EN3;t$T+lg^ZN??91YF_RycCj;<(gI*y@GsX=$!#`N=422lr6*pdrP3$O}Zt z-8an)66sR{OSzz@j=2<16t|fC;jgGxiYW6F(C@l0t6ZmCd>LWZW^kCxFR^*4V=WQn zi8JLwC*mUhCGlW0EX?5=SLUJ)_Lnl52wR^=K-+=wNl7TNfqQ7_t{NH+d!03w-*m8p zY(-+e1pukfs)eS1=#TURf{C%_F<~7g^*-->m;>kJ$R+mH_)dB^;#p#o>+|9edOqnF!}NKe>vlOmcY=PA zdXA1H*T<(Tmi8);^6wf?YJLbC7%Mk<@g=&Jj>i2i5b&=P49{#B_-s= z+a{Lc9bvBm<9R$H+g9J^099@oi&I3*phC_Bk-80xRIq9+(xEc09tc+W0etoNs_0(YE@F47c94|~tH1tg6I=7#kCKOaA?~f=Drxr8XJ^V? z=p`Qh&N9WuPkJfW!+|2dXwS@q)QGOj8rurrpJLM-qi1hl@>rm}%g4StRc z`E3Q)AmK2u#FT|udq%Ayx?vdj-C2U*k8O}15}agvS}^sOxM6Y8id#bR zV+L}IySK+IA=PQ?DlL-;ELS8>52qjlL~4ILtEsh>fd<{K+lPli>58PZ4im~#!>dmS zq8OZELF|*%dt0ReW_9;921kU)aU9s-Lw`o9l#@~ zwj1Ub;+5nbKPqFyOb@R`na?9nxmi%I^*&=K5)Fr^6UJR}SFpxQrQ3g?!OQKuCUgM2 zL}ShC(e)~2B={Jznq@bPbHtX^vfftx8OurR(&b$H9A%V_7GruHTK3h6$L`c=ElatN zhxFwE!&M|2`}eM4#xqrE-xu44IMCb6)#sO&`&;l_$%Dql2J|a8-u?bTabWlKv`)ZZ zCtoqs6X;>5%P+ev-7{_eyu!+=(10wAd%DJMF1vieUNrGHNm&}4{=a{j{ZAwOrq-KU zceb@z#P1ZS!E=l-R<;qN>g`m4vF8(30{;cX?VJ_(RWg9E-x~h2Y z9m1y~QYH!itiTxw@+LNy%Yc<$tVVNTJ7crw=bA9An!na4vw4rsD1AP?%@^lXltu<4 zUd`^IZba3yvDMo^)xx$-4G-w|PQrR6)9QnkY?#bYSuEj?+e6Q;t^Fa(?dv%Nu1mkY zeM~RJ%d9&)rqywRb5i_YM7tYAHS zW8j3dJcwY$i$0-(vn<{^H8YzOu@2Sh7T8f`ABqE9x07OFOK8|aBv-9@ccl{P`wz9Q zXV+q!PWf1{&Sx#wVz@*)#$c_Ax6TlxEEs!Fz_k!;b+c2%ls}r6Mj<(gmShO&wP%oz zco3|!gi$=Rio+7ku)c|e$3#ovJ(DZn)?g-o7R{-Nc+a^uJy9RSwOfug!PhnXwJ`Cf z`QcT|#t>|{;V*T~eNcecFdX^EDD>NUmBxL-2M@EbrWK@ewi}9fTydI;pJgl$yHm@l;Zz$J`DFQe`A)fiMQpRdG zj{Zyen-Pcj$MNA_<3O~zBhqy7SXf#gk#gx%z#5OMdIg6TaG(rpyJH$|3yOo;zq*Ye zOcv@%j6mWmNZs$_7@=A5t6U<5J%X@tY>DtQ%yX*Tz!FX$+T5w1;Mx~9mWs|z_q$%Sty<=(9hb zz_tQ#CC)xMk>LfP%pwTFevNr(MXfxw z`{{l}D|`Iz-$2dy*2!G?lav|Rp?E7hNO^*Xgr+xA=bJYVk#U3GH*XIjV~Ili9WCML zJL>HXB3`2_fbC5-Ci~um8zxyKH#vsxbp%BVS)99ANE$y|V<)J#cKf}-G*QS1I&ur%!Z$WBBG5;ZX+)GgPx;qq+OD?2 zXu3;~O6Mr5&#cpK(8=KrxV9D>I?WtfjZ-i@_Hx>yk#c3b+DCM5zE^ip7w2B;S&hM3 zvXtMdsbQEEXpcmL`Ai6|;7hUCGdGQhbLol&emv5jPq=rN2*2enBRHo~YV?)ESWxNq z`7-M%f1P}vOQY%uY&AR|$CGSQ5ILptYos7VOymsps;|Bwl(;=N7*mLD|HGb5{IxUM z{W`3c{Z1MC0auKSU2u$XAyz(y%(BolCTq!}&{X`ZR454~asL~RI6juom+4u%IWHE+ z)CrB$>w7n!p0P*5j^vxI zFG6<3YJZez7uh&+S|?GCOS1rukO35-V;jHUa8VGzOQlb=6lK@YMD-nc@hXm|>q z&2z&cr+}U2P?`UYE@w|z`t&mA@-OX1HU%hIq}dj6kZ5RGAjV{3f|&LuQqm&pQCQ=$ zC3^4q8S%BpDaO#h#h|hU!$~v(RC|HY^MErNb)>0eVaUhbtt91~?phFQVr69qZNOK3 zu|O>wt=qvs0v@mq-YFtZc^BBi??nGPt8N)=W?Q zjrrCi1_aBW;UV=GdxYMzSKyV>X|5TEqm-cjJ?G(zBK3=!!>6p~{nnl?z*%lx(?P zW%PJ(o#Zo*r!YJ6-D-RnAy**P)W28UO>=!h z)ol@qZMIJ01Yv!;k>4eOC>M&olVdJVcZ9kuh_ zF}@5Y%)`-;l>gRzFVdt)FsMXUn?!ZSY^9Pbl_N56e#(FF;QfWJ8KVTc-&H8tN$=a>cl`{emWBCyQ{ znINenE)8J%XW&`$=AdX(kec$OB-)&$2*6eSu?#8UYi&=e|J?vMb7`wrdZ8>ebo0;z z`G{@BQ(#*`^;6)f!Ux5Onex44^TZH_+_Tx%lYBd7H@p+h)~QxC;YRWAZKkKv&03VN zr%IPw?yUWEsb-C^nc(u+Uos_(3#c&~zBS9)fExRHb$g_Z>RxxE#CyuA*0eZxvJMlR zgAVJ^Q-o?QB`L#RhbYI@A{$qwhtud;pIKl3i9NEVX1Y_#ejIZG6pn!Ygep3f9pWtZ zZQ?(HV`m3?P>^+dzv#19Pn3-*qQCBMH+irZf}NN)U51Y1O%qB)_!VwCVa1eDj@$D@ z&j0}erE#+G{w`7xzduH(?+nQX`-SQCY8DQye29`BK0*lgFRk4Jm6XJ{H{x=HL2U4W ztQe*yIk+rZ#}h*6GQO^gsMikwf$rjB%1g%~C31cO*zVK__F5%md_%vmNb$5cgj6w+ zt2UH#OsQ{irPLBe&GC#i&qZ4f(t0N2M#o^_F11*8uhY%#;caF-!s8GeI50zLcQA&n zOM$*}0;3(Nd`K-SpW`(8qQyG2o{08vSuM)vd-fvf9(NX1rMnL)9LEqcsr$~<8qp)2 zPZDHuBWrhHFwL}J@)4<2;mjyx@wr!*HU|s2100)aQ35m`C%55?j{pmRBatMzd4YWb zV(=EY2}##Y%T0<*A2yQUfg@TV&lT8lWj4^^Es!s<;EA>1w?91mCi2 zWHC#nj!K$$Hm&--h36bZ{A@QFm@#br-H~f4s6O$YmWe@0E_zH|!;o?eRm;ba3W2k% z$TRam+H^>uJBu71XUmRcGS7o3ah`h8a%`|JUgcBvEiF`k<1H_DVLuIPzeh)s`LNNx?<4S3pCd&j%+ED)CpF zY2AYVP?m_qIfi)w7yJkgXxqVEv22Ymi2)kUXu{Ldsq`q)|fTU2Vf zl~niBz9sj32Pu}_0Sju$Bxm#*=$zRrJf;NxQ$@a_i~;55(~?vcHn49fj-R%jG%9hH zknmYC;6N-0+w18{?;`3ZgwKpcJ$fzaIdg`b z?@U(yW9z$RhMKACbauM6N~Pqv+k!FIr`F9^s8#n*b4(|%3el9igvaqq%BE&RE6bQC z?w&qZ9uk>kA&81L?$^agJa(e+8@6i1Vj`jn&5;oCZG?G{e9q^IiKVV>=&9 z`kC6(tWKGsCLXYK-Ik`r56#(&1j&|mMtC&)y+i#=8)QPI(JiTuq2QbddDC`M!XRKS zZq(d;Cs{TO-ozxr;`izyfG=#+MB`u4&t=8CHkv6AP1O?$j_B&cp>{nXe~%NK)s$~l zCv-hvhRSz5aWV~_#b8+nYVrutkTPs}>^(?W-AkOV0L;6+^ch3hWq60+6ph5P;V5^r zqYWC5w0B<&q&ws-Z@M|lvywA*ACs3;+x`~8F%K;N*0r_3&d-_3MBMyaz~e}y?+A5Z zVRaI0OKc_VbFte1C^^y<2NIs{c7BAq-zxEqmbEp^Mn1n&BfmnJ|?gUg*OM(88AWL;lFt5G9+fPSL*gxvaTzp%_rPhv>1L0hST)X5!@4#qv=AR`yw&Zn+)_$DyjA-%b$^Jh}I;u4Kk9;S)Gk0Dr(5v z{EHKgM_urLi9g`_e~do>aPj_c;RlUcr)6v0n6F(*TlkSwU<)92irDO@$KNh_7+*8K zP=pnoV+hrGEG{f%cQm(}8*5v{AIM!hR{$h^*J*se0cXh#Spy8)Byh%q9Rm#fbKZ`6 zznTpi1hj8aaSAKEUat;WKVF_*@0||u`n@6804%d?X6PlStBz(Xe#*%jqy+GX&|7AN zDeOF1orHfUXG_sdS2Ov+;mq?}IfiM}0A4TSYW_=}m1My!!>`M`U8<0#EIZGe?k}_D z1-q*u^(Ds~`8ce=(~3Pd8pdN@!C#qdQqy)uA4$W^%yu9M@ZRt0cPWaLLHl9TajGK7 z)^xBFKlVTCpn|5Ar8T$(y^V~}bLG%Y{#A08Z+OXfQY+R`|NUHoD?y${G;?#ibT`bDe2`XILPX;8fus(dPo`|vPgBdC zoaZPqBw0y|kMTTM+fw|rb<7=&>}qB#DM_^`!_rsoFJ~*m)1N*P^_~!Sd@Ri4(P{-M z#tQp}ZSuu&Iof=4KAUd%S&63Tws>|G(_`eTE`Tl1=(?XEE5Hzm2H(_@;6X4J?e4g} z(kmUgd{a7Lg}z{}dZjm0#q4hLRYbXLwJ7#!iy)KO8$zM8Y%A(OB>TM<-MU{|;!okH z$&WUCm@(c>aI3+pWH*so^6}+KJRD)0^|9twQboCdVUu-Y)Z(mceLbc;S`i@_RiYso z%Q&@^V3WAEL*-C)bc?O%&o#Ra?|Znn1mm*xpKO{^$DoAr*p&(xV+@+N13e=|~O z5$0LyEhD{IB3&7Y9ndlfyeBTVltRVkCf=iT|4I3Qgu4vj2#}fJ>2>ukO2X1i+&*f> z`R1Bh+d#Fhjqzy~9#$3J^s)4_yfVE(>3Zoeudj^ieD^KxD}cN2VE{W5-27uF@zXi( zyYHolCe1(|ALU&I{tbrUGJ-kh_C~lo0&c0P7Y-{05 zd!6JuG?I8?@`>6 z4etwrjzSvk>eRwahI(CmV(M+$$#MME|~-8;TBM?jUgV4EfE{DcW}X z+INSSWp?xjK~9RHk8PHVVFH87cluV)7s;B_-rozSx*^zJSqw`=tqU$wrNrFRgj12r z%;M3S`*qXxecc-6Cj((E;#*qnoLSYteOt<*oC#hNvg>vTPS$UN*zvsH1z)$_KhvwX zGw`PCr-hv3IvK_)?Vsd-_7BA9t0LL3 zpOb$Lnm*wy`W--O<+>rq54d6dRsQ_v|D5{&TjZIVmy{WWawdF8`wmprZo_TFA`=^dBw$BSPyIGj=uqcZY&i)WzD^j#eh&cs2 z&p*-2X2}|H-+yjKsg9|El7E4wsCC znxmAAgo3ftfARVsWK}O$kh#5#gM}jnv}F}@OKWHVpBKevQRw)X(<8GgxtN)|{OjW9 z|J6k(QPs`K$-(^eH&Gp;QhI7Ito4 zeF^|Og)TQc8}tqzF9py4%EI-J%FWHf$H&0|?c0Cm=H{i)<>KOIVdvuDr1)RO^eK3t zSpXFOd4q@Rzv_hE}>lovLR|EhqKlaqywjfeL? z^XcpVcd7gv?*Bt7|0NG()_;Qxw6-(npkU|wAENn3Q8c!P>g)fmEI>PBOV@uYDf&-6 zKoopzTr3@e8Q3p#q^M9KEuURoMM^6e}W^N817CsI>9tvho9&Q!@H-MJ{T7r{> z8(KF7I~xEh94>b5|E-_@c>kaMls9*<1X-~|WyizL4XxMO4rJ~^!3t#`WG-R;PkW&c z+kfPN%9;zx$$t;}JniuEO_7}8HTXSwJ@!q8L*`TZ$bew78TPQKm^7jk4UTj^L6B2M zZX)VPDC#a1LgY`Fy3d#rm+x3pJSl^JB~XOR?f9{JZszCKF_f27v}}Jc-q-zoEsrkY zaqqXWA<~7xxu81#TmARE`uzNFPH{MxckgRZ;fAg3T)2fvY6VG*N{G9^>YkUlb{<}l ze6WgsADN|h(9)z#j_E=an5HaSWnTj74*o;;F1iSofV{NSwzbJ&`CGuiU80slb*VGI zm?5ecnwK&>b`8`@SoxkhVLgJ)#Nl-NXu{;xIZ(UnUD|dT@xg)+&C4tzTLJu+wgwmD zn3u^ukIoMX5j*8l8Vh_x2WLM#8NWOj2iE8Y9x84;y4%0g77*aQmr^KzUpmg4bTuXP ze@cpjwVb&5!HH|LPf`6b$O+!JY{>glk0n+qOMz z+qQQ8#nbLaROC4)D(*wwhl;vc_hgDuDvt_hJW@{)8w@P3vJ)6}A;K{2qcDQJ>C|sV zzru4*c*DP05c38z;u;YP6A!3CYEU!65fPZ_ORPrvPMLo>esWEz^und>Y6bv+`U4%y zSW*6(Z0J=gF@`N4XmXek+)3^PJ-t-54A@ET#9Wz0rwKa(F0N+20>@k=3pD}&aUBlI2!MJjo zj|714Q8ytZ%?fhCq^=O-1$bE@T_F7t4d6T+FG&l)al(!j(9htw^*rG9IthMv$)Sgx zDt-F`ay5S74!mU_==bZ$L9l@kWI=@-Z2wu!20IG1Iw z^5u^-D8=&|aa5B58~u_gh2>m1f%8?sBTT4;i1`c#-MR(4Tk>?ZHuPv2UFbH8}0S@ift}! z>H8i}iGBLO@7)7q+czd1_thKY&=AMk#qI$j%~9jj;Vc4{PJpJ#MM4b}SlVNjO=W|) zvXUQSoNKh0!1G2S20?&#{cJ|%2BUm$pwzwuqm%~>xl|O=PVB=*j4LO>9pW(nHNE;Q zl<0JFoYH&N^VJ^wT*MSV*0rh)&dv$)`oxV-^k7ItG_l1~K~hjZt!17p`x}m*K(e(i z#t8gt<9p-%2Wk$X*iJF#ab2-eQfI@EP>BaR5H2ULdCC zA#Hr`&iU4G^6Cb~hgx3`9=bT?_!;<@Peza4(y!=fdmp;NR;bSF{|fl05@^Mma3vmb zwY->@4APqEAqb5LN{;eBQ+b-!%^mT@*gVoKdvAB2^WNz1!QJRD z^KDO`ziqFm@^s@kZHsulEZG~-n_P_drZMx>`_A{qNk=Hw95#A$F~rWk%xCZ-1XwfG z_IffxeE3FPRoUV}<2Y><@YVMo?W32Y2?V*8djkMl8VvG5QnVD2FbN}GQ)ktDf|{cn zWsw4X!cLG2I*2OwhOqwNFOe3GGh3Xa8eBbbqbNyNkGVzlC!>&T%V7UK|0Msx>8*MT&wFE_7=cj|gapYGsB zR$(>Cu)2HiU;PIz&}MeA-@#WfK>pb6AyWB*yzFUf4__S&+N{1K^#nSilW_K)ee(W$ z=(T9BcpaRhs)Okoeox;K^6MFH#*9B&^qt))>xK3n-I$@Cw_*C=IbhAr6e5>6QN7(`z`5&*hqPH@W|gn&Zx z?8H~+K49~^N|32X>*HeopzZlO$Y+mfE@Iik?-e`sU-J)@>_8h&gm-9wvnINV5x@~S zGQ{BZyHR;T2>v9h&-ast5@}Mg2d8ofz8twrG7IUi32$Nc5?EugG_QqT&=;kkM1C~7 zw5mnmoj2MK0)6j9RT6G&|pLHZ2B)1oKe`ZIVmL@C3<*OOVt^LfRo2hrCn zuU0SRsAnu7sRQGNPCpC5yL7G@09TOX7kqMSuPMyFYh*@(ns5M#gXV!uf8YTzfJ%v+ z7Ow_L23gn!iZ7TQ`f3v_0*Qn4O!TQT6^c`TbAsgF;0*-wPJ2v0wE9gs;1{g$Y4;Tv z<67|kP52E0o?P#{*0hEF(gbua3D1#_7b*b{y!VMNvV^R zyB?Rc)n7s1Is8BHV?blO?2m!nKe%7tg1C1Cee-^r=9nuQ*c*92t=!+O0pEI&jAkJ0SG(ac_9I;B$>by@yx;KMAq(Sex{7YOus+AN?21bnRXkB~rTDs$(4 zx-9zV_uc5*+qaQd1Ow3yhKG4Bywb_R_N)5+a6o$Ae$>l0^Ca2m?7^81imwgG%EZmn z+Yd!w^8@%m6=dB%Ie7F*W?BhM&0G(JSsZ)bD*kBqDGnpLzIL@=7I_e-88M!a4$FKB z##^w(C`kTsh9le<#+xxS#>r2HtX|s^9ykSh(V;deE@q^uFun~Z>@;6%KKHWMu(@tC zvA5%SdJ&EU3;tq#lsq5A`#eZL;h*-&G$LJ~7pWvO1I|#wNU1yh_nM%QkCwqGkmVK# z_$FQpO5Z@iA(y<)f!?P-bl*82qyCY;c|d|(0xpF;Cp$#nCYLWKz5ORUY@gph?yLT3 zuW;&@0Tw+Rn2((wpL=+tar1e-)Q;GDJWkCoxVcozpSUp=T7JjoClX)}56?t?xug>o zk+_YGEIhpp{{iuK_^bHYuk%G%4S+ZQB8-TEB4H>4;Ql!!VVvF-s zgG&=W4PxBMxsX?NCO&Mk=t$SIT9=SxOy)@9aP*L(I+1M(+bE`yTsus=OsDRj{FHq} zkyB;cZrfbjZCg2~%w~LBa@*uSuG5SMwnKR9dY`zRS607yQ-&Q=3#d*d-DIjwRWn^< z4WOoB%eHgyIsXbuC(JI+F7JwLBdZ0Z1;Oq|a!Rg=B37navRbxUKKm!-G3BjlzU;KZ z$l^@v?J~Tv-C{I%T0qCq?S8GHaV1aB#qCSWVK&>v^W^*J?C#m4`x3>`@+8%UVq<*c z5LWvdt!dK;$cwHOn9e=w79cr9=oIM@($+S;VSO_`%iYoWHL zhKA9!Ss^qD-aP5WCCfg}tWA$amqrz*Ov`~l_rj=cfAMQlu2J7iZ^fn^u}Rl-lW^D> z%}H(3`Tj<;ws^()!uG1AraRRegw7q4j-zA6d1lDd0Lh-qS(;a7&UxHIqVVX_VlLjj`qAAh(DQ!h8FY>+gg8EYQfBi^A*{pTZ+=_zjO& z>T~s}*7m_U7W}OzSD%H1Z{9gUCz8?EP91m4{%W27j1`2~L>O)i54P`;M>6i)zrKvvf9%L>!fg6df<`Q_`{5c1=R`tU9Iuz@N5TNfLF^%c z6eqI#ke8{Vs{2w#c`r`xaME_;`Skh6GS2c##f{odQIF17$Gg;1(79Zje~w~I%3o z-9yOW`>G|ZXK|053d~Bln=%xiL}l0RI<=UJsJlj*KpfC3ec-PUo1m?I*lh&(VG_6D zbD`#6Fq=SSyWUd^?)}srK(Pbvx4&#d2zCt|vGM;N zkjpL+bs;(SZDP!!-~F$(;Pz9YLNab-!N)_Qd&W;tfh38sB2yO6Xj`JVq#9Ek&p*6U zxdYipC^7ZE-+k$;yF|s)f{*rP!4v z+==2K<++6dfdBtk|* z>;{qs+>F9*n*chi47?0ov|aRFF*Z+ld-1mgxAnK~du@)woK!v(J_@~MKZOEwi;oiT zC|?=h!u}-|r&{YIr@_C3;yPjarTB>XsqN7TP+G&|qI8Ds_EGke+Nx|Zw)WxcLEMjX z_80f*@31y4+@ic4^R_9jG4jyzP&$WChB1aPhSP@9hBbyXhF6AGhB<~fhFgbPhkGL- zhJyBa_B;2z_doWD?m+Gw+GKnw{YdeV;vxy6^1|~XM~8Z&@*?u048li;{lg6+4I(Ip z>V_SL9EOwj1%_{iRQ6x?dxr>y7Wd;0UG76(lU^HNqh6~##Cc(O@`G9uXg#pq`lCE} z&VzGqk!izD)=tz1OpPiPiDi+@Az4CbLX?d%8S(r`4HoR5xnMRyD4fjrp<9!9+fojb zLJrwC8LkPV@}oX8vPpkXF~a5c;fxy<$)}KOk!#T>hS7|x+9bP_bSbw<>CrVJtA>`1 zv)aVFw7S&0RCH;!iMJzp?(KXOdK5OpEixf)OA(Ob6skzDB;>B7J}e{5N0!3>3;RWI1}?p`lR7(PuEbu0LY2)X~yq zxh^jstxVNA_GM2{TRd%IuXMdNyFS6iz+`6XXk*ZKxZSz>zdt<5q|$d9y&0%)dMr+; zRc-v;?0QbsS=n;?z;mzNaa`GY+4SBS`5;(b`rZ{m5V$`*9&O?6am~6NN>Z?Mzd1eK ze0!|5=y1DP>vZ^OoOKoW8Hi%&{+y_)>;4|ea_IhA0fE@dao_R&nS=1>`P`F`i1qs% zFT9?D@caUEu(oiQ2p% zeyeW(+F@X}+hH|-+~V~?(~C25P&Q1vRfV%r*QH-4q5d^-V`(yRcMD;UtXtmUqo{n{ z!EeF^bsFfgkmF3?H96`-VfXog%$9*j;7fan0DGTLk1~2XG`CaWcv!&+yGVykAz2eB zit-A_M|kK=LDdO7Gy}!c*k}er>^3l+f}#mM=8`4=;kq@QjErm2UCN!yCZfO$UiN^N zAuqv54fc)tgOMpl7F>0Ld$C*V)}~ni7aG-jS9gUg$OY*{Zjrub!%^<&G>Ut#gdZgoN)_nNzqA707wx;?y zImdoHa~4!qii6+GtFXEXlY*Pg!RYWx({z>KifD7Y(Yf)zpN^A#TVF6m(I)H`We#et zyG~!3C4{EnNt)#JHnFVINFZ8@ORgD%Utv2*C^d;51HYgw9B|4wd5R*-p0!gMWA-Tw z1>26KvDm0$M!8~45l$p6t8gQ1U#Uqbk7|YE+BVnk(Ck9zt8>RY$#C$IHvEQ`^pX7U zHv-wgoM;&=%1Y=j3rY5I;FyV736k^e z@XU%9=RehHTy8Vzd8YdRc&{ZZ8)$`c&Epv=C{Vd9LjPU3a)8R~*G_%Y@fxPtXG(Ox zmoaG5Nm~6d01y#Z1I(MUL_dWZ7gzLo>3TdSs+HRTHNy3FcaK111|MVF>>dCRLdfx*e@2%Qr)6rD0|v`QPb{WVHuYR)K@j`)U) zInPm4ntsR0V`<{E}riQPFX>4;?&O?KMKciDzJ zY{(H5#Wk~`tH0u{A8j>~uj~DG!ICcwjg}Ak8t6uph1BfY8raGKH!1CwX)9ylq+QH) zK>RBL1q{~8h}3~Z$qL!puM8S_IgDpD$n4HisDhDfCHT!oNJ1-un2(nioOT5n?JsAc z33~C(fY(o?d#!+nrNv*kXgEVV{_ki3jFc_RHxvZMVEe=f=|r}#IplKt3vcYH$sHVr zIc?9wb1au4co5NcW;Dr-d>@zyx{yo9Mk3xJ`)_Rjc*YyIQJ3K{ZND&~DbZiZ^H|xa zE)76Yshg2n8GFE4YvmwfZaV=2f;A)Y3iBbe)}I#`Zb=D4mW=HRxGCMVx>31$yg*_N zKDYV^fh%EO2)Ez^G-xKoK)Z(!O-Kn{QrL6I*55a1*Sof@jrxvV^>~?ku&&_XW^N}o zA@!^0BP#wJz&Lh#7W-qeV+yF?BH+McMfCsdCO?q6=R#F+Ix67w7DH>TMCjH9LryN2 zR~xWB7eX%bmHxd9M0O%Nms?2)yp zocH{xt}kI0*?(@NJ~CzF*=qm|$zv6%D^>}Xz!CTwd^wBzH()T5bcAOZrU63x2u*vS ze?0Rf=pQf`#4Znf)4v5b^CDBdkH1o}{8aiKXkP31FRw}=c$`qZF?`eZOx~CtIo;T^ z{9v=<3x6~CA~7rOlB^FmF8DnKNPqFR+?6K6CSuxw$N6yN7|Q=Z zw-AX8Tj@V&7bPL+Mhlcu%@+le{twd11tXx7{{#B|$D03w(jR0p%Kt#O5R^<>@jqx2 zg_2Ir7lld#6b0jvT~ZpIAiE|1uUvN^_P^XE0ycR}@ITl^#-dCJ{15ucm=%dZ$e7!L zqmh}V$0&(hps>hHljD_ykN-~%B8{my5P%F$ZzKryKl45^*iC^NNZEZ%Xt?vQ5gpV`Tee$N!n@4#fPIyF|DpTT1)~C&*Wn^RfTI z4B5KpZVNGE#1B8^kIl*)j${Qmhek&PeAGb9?OoW@3tN#rt9hjVCO59R#H5h=m#|6q(22bb{xm&XK%*mw5r`0Xht@o9! zy-TH0SZUMW|3Re7Si;I_&if*Z^Ew$_ZO#LahKbJ8d{d#JcRJ;Q$O zsb+j3fpcI%?D=gD(HnZ-0m+TfR$MbSJ}5pSKGaYIaNA*e$k04WrFShNDGVSUvxN*d zBib0)N0d-YVWRKlZ=?#@Pe4-ihHuB2_l|rXCa}^X)ha~j)xhy<0!3dT* zJgPQ1G}J^5zeZ=%BO5EI=PB2z7?j|b;TOefj8;HYm@JFSm5<~@Eu6A($A}j#gkx6% z#nexFPvs8uurgP|!d8erqjK5hUyk<}3`~NNVMC_{@yJq^4lj-*7`wZZq0T033CKm= z5Oedo$=n@;^E6>bmc_Zm$x>Y_dzW>%C`WFcCss+jBRb1q{p~obqz7J<=?qU(+rpf_ zZsmd8Y-9K0Kt%RDSZ`ovl%SnrISGk{!%|aq43r4Dmp${&_(-b1h8td#-qJCd+{}o+ z9Z5y&bTDk)*k~_$LIvRu&)BwhV zjKZbSB`b}EK^2qMZY<&P~B=#&0EAovxxa2pBxG zk8wwDOB=!%;|cBd2t3BndH}rs!MioydtAU{SkD2&8@wEZKEMMr%kgY4c>@{I-r`ZW zx(4H(G|Ne8sK38wn9H8cd|1_dv#s`FSI^~(i&Hx_b}~@z5CJ)uxCNecqJWz-g4Gb! znH&+r#2#%3*ybCt)<^D^0BryRzU8iqBRGK&9+6uC9N7`a7q``s;4Gg3eeJpr&U@0CaN*n+QV0ZeYh#88!kS{V&8 zQHK=ggI#fVxmH+pR_e*EsTGv4P_s8030&{WAl>3iV7L}Pd@$=RVOf~Gq@~V0+7gU> zPz~b!ySBqEV1W?BYIy_jk-|l|wzyhdIIr7XSzU2aS8p|A(a>eBsjnyWXa%GuoA(mM z+fs>`6lvRbLGTzGzx7>3OG;76Sdnk)_p-6y;dHM6rlsqqyFSo+MSPy7cTJwzN}Jw8 zw9$~ZS0}{clb;1)!$t|kWrtu;J$i40MvuP!G&9Nejx$K%j+K%ekd}Yka}L zU51>j@6{a(^SOE0#+MAu8g^YUx>|I^xg?`@C|L;^&EvEzqS{92$`d9|?jO7bJx>7{`-goM+96(1nHVos-t*`13t5B~ z7Q9HFo>Qn{T=c!X{3Z6u4^*Sy-UIGra(ioEZQo=;EtmJ^Ya8R%`U`eG!S~RUN~Pmy z6K)%#TOknt2`dymBlut{u($8wmfHv6r%uIb8RrdSd6C*nuuWl=s*3^n&Z)_rcXdvj z?w0a$e`f)Y5cxSzNP3KS;UC%cl4P?a0~J&3TsMQvDcf*Dc)0}jV(d3DBb~7H=+QY6uc39GHmOV&1vAH z^C{UOrCsD!&c~O<=kAgF*mdAL@&P|0kNBJiDP}p3!$^%EaJ`6cp-$!~HO^c6?c zIL5{)&7viCO{$QX_K(4`$tJsX$HfzYaa;ml4|da*zsYysKZRHw#e5`rXZi~Lrj!)U zW)#*Wmt+;FB%fal#jxN5KLq_Nm!F=JFhj7Rj`-#LR2wFT1p)@9wDu1C=7L6Pi zbtY@iO88@7Ahrp86wM(IKAL6$FEk=-Y(SL>4RlJJq;r0OBjBU;}3 zAUfl!(D9NuPw-MzMJtI^DycfIXesAcsahggB0FC`e_0l=#Fm1U5}!W$Hb7mJ)C~$c zRkub>OH9&4iqeQQ%=UzSH^Q!sBp_6{%{Ta?$#ae_PcqC%UeOR`{~QRq2D5f(rde4C zoclgxGA9(fV6q?eh|DQUEm{1UuE@|WR0Du(oagR{yCyp@q?j&YITHsObK1O*HR&>d zaZh|J#%f7OGG%SgJ&Z58<&QxuO+bsk@C}a?l_8H;3Bc}yGe*;W7~QEzQgTF<_rOO4 z85S_{JO2ZBI#02ya?hDfl=z}EWe;pl0%4w!#kB1Jn))>1ltVZCgc*TVtnT{Dtzk#L z!bR0&_+l6J6Plu!4^!!YG0E@DDFPup&)cQTs%^kAr5|0_oz0P|HvIsT?4y24TH`=^ z#fW}IxI>W`Lx1q4=9Wk+=;C%k zAot{^n^$cP&H#m_K0#I&fuba2f!nI8NySUuONR}%rW^QrF#H>~NGpt`JlK8L<~74| zjw2x+TC4mD;s}t~>kTWS2d_>%_(j69z>_ zo~=X{Ti5RwIX}~w;LYN7%TSDBQ^{yUG04GrL&%Kmuu)++n#*qZ_gp+Dc0gcmUB6a? z`Zxrb8Rg?0t2US#*9bHj;2(0h6_R0%j0)*c(2WufG?Ku14>_T>3k~;26O+tWp0e1B z+*3k14M6aM$?&)J;2I4sEgq~~jB(}wH#liL{wgDq`&GaB2LkZ)17Cgz9B?83dbShv z8i}Af{^K&Yi)MfWou73G@>R_bX*#zEA;iWHnG0=*c@h*495C1aYftGHQqkJZWIH6r z9;vER^__qy2~&zEn<%fcBgq zI)}``6vQ=rt8(4GDHXcdjxu{BjK2_y?*lst2-Os^eI+Cmj-&nkvbrYFmN;zxj1$Xq zdj}dZRF8bQ_nY+8(wI?GFckJM7WIUlwahT>2xV4idnjp;In>0!9l$WkYe4-oq3I>R z9^`c{d2Fxfjj++oRn(vwbXQ<*&}I!`VNr4U+P7fj9l-{*amxgnIBaSKpw|;Xb}r`q z;!T&x5pGs3%BcE~hV0Q5D;Jjm%~7YRYz`jFX9q4<(^g6*OHoQx#m?cQcB_tsLqbH$ zDrr;_QjEAPlX@mh$&@aX6fEM%D9n%Iz63%a=q1$=2VD?DxzvtIHMLB$s&GS)`-Q~S z)F4_s_D>V{Ycb}t!l(c29_-j92|5StN6_UM+xfwnA{Q14FD2KTo{Y@Bgq79o=!u~Q z*VoOY!={ZN(bx zR<7FU49;r-~|;kGt{x$Nfq{5LWasZPMAb zIcSUupGh7$?PS&(bM%8%fpwVQ=C2Egq(kMVhOy0}F%5<{oMgyINu>*+>p{Ye!ci1q z35?!tTr|~Duq_l4#b7W2+cDWO%Tfg_6=JDi+0MG+xG_<()+-5yIbvxAD8Ka{kL52C z<$AOj(mfL8GUc`%-w%PW)y8<-xl2BylDm?Krbc0uo$I|c!IP8jofGZglp^k&pjdfm zSS^MewOFMh{*{%EBB!@gNL*{e_mr$cSV-?-sW&WNsUNiDz>p}WP{wDVrkB*F@gT$S3WLNCh z>@aE4rnQfOq9P);fkHxvc82)y;Bul1Zfr&})hajvQBm8rae#kBiTV)x2z^wFFb%}V=ap_Fp%HB5!W*XVy& zP%1l8~X}I~)7vRXX(Njx_us!!OVlaQrH_54!wjo;k=ywEt6bE5APCCN6dlozowDcg!J9nGf= z%L4G2e6S1gkhD-e&5T_SH;1y29pbu3tX-C_Pc@44qyAQ@2JRtMj)jvh$KN)Li9M2c zXgxKZqosVeS0li&vJ<&zN?IHK;WM*T|HEg`$WLt2h>h%pY$>cntCNEz)UTB`gdEN` z)M4TugJxo)GVTa4q1zCkoC2(y2;WyfTY~UUDQW$sU%ziE{|j-hthHo}GrBs(UE<@+LJCE4LM@=Y+%gh#Qpk}1lOca8|;veGP5ank}7%yEb z1y3@J#J!%-NOfNvN-D$1rnRu)9oBgNV;4!mbY{G2@-5G$mwONPVmKj{8Z&Ppj7ZL- z3w6rTf|)2DjXC2(fk`5({-~BHjYg1`p2KD=3Sav|+wuyfL%uf>zL%lNMx4djO?896 zx}ruay1{Dzfw?qb7MCp0;D7Af+Up}GC35%Mc{;of#aSArmJ-(tKdDf%D zzUMA7JzG*vNollF%O=Xi)_E#f~nu3+2X zau8VjNYnsZPQ;?p{M=l8iZyNcvs&FO$^>{c*h~T=ayW33y}z|1~tF%8ysl{5J@N;-wpQ|*Mj zLu*3YH^tergs)gr`BzOsHA|04*%K?d3?Kr$PR}~@%d)=smazy*j&;DIF@@CGa})-D zwtY+E3J_*4r>Xm$hUSvPA>}=j)3Nwaa-uOyH7lW!VplvKCvU&mWVaIZ(uijcKEsD_ zz>EBCpQntii>{5XgmMY(DyHk8kL!uiE%Ot(hxV`oOOUQmE^jFxoCEnIhQ`ipYx;Ur zkS2keG5JW}?e*JIUNlxqlavlxpK1%Wi`!Y|)C8$h#9hoX$m3JYg6Z>jITMYg2VqqN zZt}2`qon#SKJor|`7k~<@-H`2FrNGVTUX2wZ^+y#fE0TLR27-JB!a1EzN<4LRAi*O zPE3MQ3BO8%-bvYg1ci!xo~+RZ+%-Fu?TVDprA)u2ikXR#5gW0Du@Z!(RJ52=`bk#) z#CYvR+6+SlJjF=zqC?)Tj|j1Rk7Kr97MI<-oz9CxS0&Nc8iph>Qbgt|zKK&(T#JnG z1{P5$Zx;?MCWjbM(e?Z`PFp`f%uBXaqmu(bBVELlIC)x9NxXF^qyy|;;U?By=`HsY zcn$c*c$y6iWa?+ydp=#QcJx-1RCj!J*@GAeakS47^2CM6KwNBglbtRg6G9VW9mADg zjk_Q&Y8<;&Y$#I)36n6wg?MtAb5-bDlkW{fdaAy zBtAOkDJ?=w<1aB=85eQ5rXu#@)f>v*Z+*r|nx(%=dq}lhZ5`$q!hg5ogX&@NUrzIgb zX3Po!!~vX;CQJqD8K+pr_)f;1Cs{apL$BLX$d%j{7^yrX)6&SbIi9Ze^h7F)H@>%L z8~o(hiTj?*jX&>fX$k8Wsj&n*Q-9~LJVq!7agl-Wo|6rYiHgP?xi=YOqha2HKc$e@BJO zjA>igb1aHKP_cZ?M7JWF7>jbc)7`g(C8_L@@-8H&T};x*QBCZjROBTi3#0(xo)62F zh{$u3*k~zCq{(S3U<>!s^%`E0m6#H3b((kTot&#rWIQLx#GVHE+H7)pdd?a~)LL^p zW8l4Q)!q8fvu1pwl1B3PTk%NOrmg4g=L*d|i8^>YTSR5jthUzeuY|7LwZ{u<=IEDO zOhQP@mprbJ{{*h?5#oxBJ5Lo+OO>Fzs>WPI?>JXOX&D!bXK3v3-X;f7fck~AQ}W2D z{A+4kcaOREks=<}6McNl?$(v%{Yd!XA7RUG!6Ci$hFnyjPB>*Ar*oBoD7hgWCsvb` zDRHZgSH$W-7L>zxFbmWq?OMUcuQ znNiN6g61(wQeVF#x+>;N6CesYQYK;oZ9WNht&ZQc>tn>1kCLp!!}Df5i3j9n4|Mw& zf?~>J(<6d@MyI$db4h4Hv~6Nv!+_RLJ(vw7E1_sB&&4lfzD(TY*cJrkI>yp!&F|HR zN8q@})@j{Q9KTeggWL>ffbYXQr+qcljiqJg4f?@j7?c?7&2uY9E-f$*P60BI%#J%x zw<_{r{tgndnKQSl+xFnTEdCczVT|@xI;zOoKmeX7BO?-wYE-(RzT3Z0XZ8gep;`hg>4n5URiX z>DEYR;$I4zR4pyH^8X|40=$sBENDSX}}~jZ+h+&ZqNF4{u?i{mr(@ zp_Bre7>?mDGNG&_&2nn!HeN?(#{^g1Y;JlYDkh0MyXn};*jahSs^7}tf34wV7eSX%eC*H)sH8!wQcy@VW6aKxkO2wG!qm5hL!5)e@kxl@B;lql z9@53e^Sc3%JZ^YQo2H|Fa&DJPrn58S%=J9+EKu7kSL*?DUAz6Sf(SmHG>*yJ)xp^YyY3!zBOc94~4GW7qz88skpw7tOW*JJv+Vs>;4b56e8e1*u19IVTYUeW<|0<$pz!g2dn#%HxS4UMghe^i zg$JXDP^DvrXf2nyo42QM@nB2ck~T@&gYMn85hp#HWnY;Fn5#m zdA_yPf$Xm3A*3QdbAHGcFmT!`|8)h&5l_i`33m;H4}kLY{uq0*x85(vi~%C9mtLW62$E-MG=!rQa!Ys z5Q#7FcQ3pC#J7P|HhMp0chwwndPyCsHkf&eR>AUZBDF{XTYR~G>72GD%x2*n_Z=8N za}kK4yke>FMFIDzdUqDPbG;sq9VOnJzXj%ID9l1l6lVNGW0H`%hjRaB*m? zqGgwF=rL6j==Mqvc%--xg=9&ul;Zc2h%e`Eup>4cLb95cTNC|yfQL(qu^P^YcX{#MDRQ8qyq zlp1BLv-GBu)3Ck|u;wmeSFdDIqiKUSo=MBwp#8jC=K1?J&Gz!IpBNBaJ=DcLzpY4o zd|nj!em~9sb7lO0PfuFnwJztOX0BBRE&^#1SZmkK5yLXq95>}|6sr_u$l)LSM*J8g zEpB&En>zMCX-c>_P_rzZMaYI9^(x|0E)a{K=Km3A z^n0KT5%+V!E+gUkVGS|=OmpVW91dvLDqsSia^b?8e9EYvlO@Qjqs#Vh7H0Xr;L7PU z?(35JZ`$3FJ7aD1Gg`RFcWQ$(XXL|Er9ewymPvbY_B+$>dF3%Ht#qgB@$2>Jzwuf4 z@?pkJ8!GN7%*L6ITn`&ZA{&M+uMiAgE@7$EW{MeCLSDSK82Cg#Lu4XHH-R8l8L-0h zoyJ%4Vog5CYFlySA2vU_810IJ_%VV5UUZuSWCfivZu6j2h<>M{N zB?M<}FzWr=NCxJlcF%;Vs+E$2^#M!Wc@2GAf7aV5{cNev)sf1@>d==ZhPqvS+K2V; zz-txf^V!aXMA94NSF`iow+^5?Hx1*i$0m6`2igJTW|E_&w%DTGMwaK$U*{`&T$JyW#Z4ZJzv`nO7Z*iPqZxIU7mAk# z<0jhb``ZqU)cnu7`WHzC!F2S)VbNvizBf?`;65|MEg_N3zI=~0W)Lqa4BV#ngVJW- z^w-VWGmB#@84TDL?Zpn{Uqu_sI9kjt22~!Vu{MLoizlt-W@IbR=Srh8zVjQ-`aM?s z4t^;Um@Xa&{}!VLV!Q|i1K4)ZLf=T~W~n}|r+dI`OWkP67}bM$8QIjYW7+b($f+q= z+Fo9&7S%G~ziA{zcw{ z>_cNI*{i?c<(&~;zoqem8uvtQcC~W%nz3_n@?*9A@@8I}2$v%rdoiijqi?aBODF=B zmPV8i$AOfW=R1DzD_zCxCnU&bX^d|2nI6Ny;b;4#0XgdF>%QDyx8=R6 ztwit3t-92&taeHNy425U!(IN>M26|=+~eoF%^GuVhxq#EZ8PhuF+u?2#fhD;j$yXR zIjZs7#_Qx@vC9AV3f0<%h5t^awd#Hm!z4mkFy~`@`sej_f%l6C5iB6LIQ5Tb=#`DXIj=+2M zvO9O}q3^dF-Ur_?=j`9*pOh9~_3tZ4kXdvPHfL&zWAsg$FFt@CIf2b$z*Qm%5sb5= zMe-dNtc;b%$(ZoiFSprTpkZhW@D}*NRN9c4z`aWZ3gYMh+pauD!|s(^7b%k1wAdT5 zPrl@v8xRd2vRR+rRt3h&fVvs6Zl}EsFg@uOGj6#fto7C~dF2^xxCyG<8igJC^&uVI zO_0UgwN?i}xp6CvoAK+-qx{L|W!-t%L1TyoO9mr_Xqcn*k!!+1w92|Rfl(QaxrjnvFI8i-J6;^T&nbr zO`P9wD%=;cxqlTmCag1$pq?l>Y?sQTS$$05Qyy$=jw))~+32S4RV^ADtqK~geMEet z8e?4-nl$a*@)7@bRvuAXG}_qvR%Y)UIxkJ6j2$@|ndpfmsnlB4xGZ&M>KsmBOC%LF zfPrP|=%!3N%fuAvkF*Rsi!@&jC}C%1tIbGTxVVU@G+vh1CE~66+nB{%Dt9<9Ptw&q z$$KVLm4FZDXl^JD7h~yN^f4)J2TuHxph4%l(9xAYtQ9S6wz^Zy$SO7}XdDk*w6SvL z);Ab3D>7cK0f9U+-D$9u3 zx2<$pc8M(weOs*od8)G=4m@u+SEC=@1pnJpJ`%!G?ev$uPz?L9y*5OCrmy=l-ew-x zIS0i~z@pn0clKD@S+oz1(Hl`JsiMg^XF{tR`No6RlTDfslRfg--Doz&mIkty` z$X7l0I!Ch;T(eq{MIj>0+Ef7v%TYgyUWy#tl#mFg47N!IXU?L|;hEg&7q`Ts5BrHg zq;K}LF02TJIBp!ZX%3Sdy;IXz#QkH${o(m}fi9bcU89QltHDClQ*YYE!1j_2$1CcdvuxeewGVF-m&_9^!7Jo8)Cl!#9fUu*254 zn7nWni zG#Bi1<@NnMyLRE(#WTXqbPmnPoE!W)YhB{wgm}B+>xmVM|Gijs?C2)_C!*ZEG99;* zol5UNi8vR1;Z@GF!wK8gl$c%BHqHq8^6bo6Q{J^(LnT&o$DVWDr}BOFOJln)ON2kW zg~lhnnXA0&=;gvx@A>^VkH~agyDI)}+Uh&n3sSh3D$5+dXf(_4%#3OJ%TG4Dr#GkjJXP*fD%?55XR-AwGz%-$!${k^K1vwAG zC*QwZ*TA;ORYto{@OZcKk%^09_qgbYx${n#Q|A~hX6)0J`1e56efQGO?0t?-4~!M% z`lFcFUISip`RhMJAIpWkm`4PGkI{jiTY~SP5@Q3@yXj*mp3gdLz;o<<9hdxWXU{pu zV%Zfhq-sCeV6StVdC9g*l48d#-YMqWM`u~jP;8vP#DD&TP0MPt)^gfZf1mtH;lsg* zw~6!4#WC6+IvJtHwXT;-{Lp@>>f@U<-c7SSaN|~DQs~jjZ-SePKXAvt{aVsuJ#B9# z@4Xws@wKM0Pu{(`!{3v@d_FR^nAiEumrJ{^*Rjl7m~;2l4D-ZD_p|XwpFe64Khd-- zaB|ztuUs|x{s&gs?3)$d{v(Sm?EmR}&r+EyeKQ|4yJNtP2; zv?iRAy%Z)S+adL3Ps#M~^|v#QGS7T_GS&OmC$Ae#Yfig!nTgy<4Cs2zBC%wm>PgkJ z#rk@}6Vjs`tghtle)Xm{Mkh|#d{$Ye&K2@jmqJr#WC%QM5LwL7$ie`4Y!QkW z&@CoL=;j$)8kiYir~{q6gQC~a)X)TYND-Qtu@So849$V+(EJMwD>G9xJAh&q7U=#3 znvU)dkcDPw<{21RnpvQQJy6UX(>&nOOlbBR7#bLuqMK)EUO2*qzej;R^&Y#|VU z#W_$f@X#eRF(X4W4D*bP3^439v9Pd2_pgbiu@QP4nOd5o$2l;93@p*&7np{O(BlZ` zK41)^xzErL)4xW*gQn2UGXkbHbhnsTS{R`F*VMqk7+sy2fw2L)y=Dd`mPqPKiV}fm xO%;I-2@B4wN(E&>{h<8(65#nx;KQeYXK@vmBo>u`b0{zo0W%?&s;aBM8vrN_85{rr literal 0 HcmV?d00001 diff --git a/uncore/doc/TileLink3.0Specification.pdf b/uncore/doc/TileLink3.0Specification.pdf deleted file mode 100644 index 363f9b03d037b340981ea2f27c752bbd2741a20c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123869 zcmb5U1ymg0(&!u9Ew~Tv4uiY9LkJLp+u%cR3GS}Jf;$9vcb7nL2<}cG_#6J;Ip00^ z-uK?Rcdc1_Pghsfu3c5#J+t~pqb4Q8!okXoOtX1-{1cgj2fz++FtI@v5@OTxa5M+7 zNg9KVK@OJ4Y-+}q<}LsZ$axK95fNl_d$Tt^4&eW4@TT{s?QCuj{+BwNhPjJ_tFtM@ z9^zNc*}+uP9P9$nXM-#bVAC>p2LssT?I3m%|C%NLHOm_!vq`&yWi`RZVDmRyS#|&i z@4rJ}lW~A70g=i;q@!0@$=Q?pv-=H}+cYU$u$ z2{LCjb+BV|bq0w#3BR4A;S#6el!1J}kmb#tSxrq@Eg@+zwPH0lbA^Z<%`99Xld`;| zhAKC=k%GC4i@A}6xhpH=1g(p)uoSI}J=jy2gVxo>QdnF@-9}PHR$Ix<(ZmGgqNU=j z;r5Z!MZ);Q7k(o(NfT{#hPSGUh(H3DgrpTxJQo1x|61*T1pjIe8EX(2k`6W*5F{um zb5jR1NZFOl?JdDp0Dg8(4u~&+3)tD**bdn}GuiOC%zVV|Wkc2J6JW))OAOKMgz!~` zU>GK75z`Sc`6zi)`fIsdVR_cN?NZ4lrbvOtx)faf7x!RFRQf?S=kQqc^>*9O?>8L_ z?Mu(b>+z7E_cMm?-=k#X4L^#-zrmufFZ@R})BMqX8CeAy`MzF;ZuII#7)947rTpnnWwWfqK8sR7m6&$0+9PKA|;;ew6`xSWO_iEFSWXVO!g4RToLmf9{IN zB?+y77HW2x3AokQID-Qa#%`_iB96XLXxK%g=7Hk;Y1@CcnIPd2JcAxbFJ|I1kM{*! z{##w)o=4^w{ZON-+CbtK$QV7%awgNzX@b<&KIeL1k&wA~(H>OJq#)Dg8zE zm590ID70xhpl+y@1CD(qVVOI8sMHRiU?acI=;VtKuc3cro0Ro z^_Vg3mqhuIm1nN9Zy-}nY9Hd``~IyBDdmGdJp=9GQRxXFAwSt2OL0XkVzuNwn@z?e zqsFv+4G=HO5|_cyfv`dKCS}GFSH2Z%SLUP4p%im|LM!jj@TN;L1SiX|8nCOAqT&=990KkGig=SkJi$e4!GcPaQv zw47poWSRlu;!g!uX#Ii#+#I7}Gwk&{)kGw8A`yy+`^HD0aIS_(;_4V!?~&pct^oG- z88w<{DE+c$#PK7TG65f(lJRrfbL4!JOQ&VbQA5F{V!8XJGaaIMtIBb$Gqb!zwZIAM zM_UHC`r0&=Mv|%)w1TH{oGvK~w9+m#DPLr;%hfI<5fo=GhE>F2PtLNRB5YwiSg}N8 zo*A27uK&od;@vYKn_t(4G-PP5i4}+q$%MJpw)0}jTRuoIJ)6SZOHsVMki>Dw;fVKx zYE2qxEQ!PB6cT|)cB1j$U8;=01*il@@B3m#RcDrfbhe*7AEDHbLeFGtZy>oiP`Wc+ zOI4GiwNFWM!!*wlU&p{~7z=7oC0$VUWX}dD)W9y66g5{<{sAnxn^5hp$B|LzW=-bi zBDiEc%3S&SN9&GIFN!7n@Y!6yC{9&U6yANLYMP-DPJ3iU)IP}28k%j2kZyACO;Ng> z?`yj9AUV*#FBqeC)x?OSsk`yF!;JPlgj!k6{4)zv^qe9=D|z`&{Z2WGnqLjKNcx+J z{@o843GQ+8MpAs&K$R_z%*pWKPjuO-J6U1g?VbU1qMiJsTGd8oJ{O!D20kA8JC>*e z+{z)d^IZfs9~n8OB6ELg!8I?W>o-2@u=R~XS?_9`G`1}LEL^n9$CV*t`m%yf@?$Ah zyLQ pO-J;f%oH7lIvKd%zVF`ps9`E2dM)y7+}7a7o6ZImrk=cR_RNbe3eX)`+06 zj{NAbtuyW)mN=ZynjF1#5~RHssNBvVyd9#ed{Dh2`An{_jZ_i=`s^sSn}CY4p!NJI z7bYv}z=~CzR)lteKn{3EFR^cr#43NaoOadxp7ukh*%HpXmK;BH$-1Qm-mANmbC|rd z3q0MwM^L4s>uF5h4PQ$z@KfGpIjtN7)Th$mrY9DpEbQ;&736hKzl^w%aC6 z%*oqa=^Z1d?1>Cnl4>XG(stVF1XUtP!681HLz`P)uhY@lHUyc&+|!Xi>iHh+#5R=0 zxO%`XMOAck1b~(mL6srd9ombZ3l#-ce$hWFgXJw(ih_GQ0}v?_l(qf#;A!`b^>>QH z{-$Zo))kNv_qr^hSH$;%E|AGE?XIR25$(#$?{<4F2;aBGPRUKhq4v!cdmnO`4mRiD7NMCE2kxcHcpOg!8&J$jHTKXgfH zb^!!neu)4C2H5dzBP#_JKPe3A6%H1fHEIVGaamGCV8hE-3`UY}efLU-p_!xvAr769 zT75dI@j=n!g2F$Ffr zbm!K3VCA<>u?m0dKiCs<%Ip9cBlL8~*YpafbnR)*OX$h#s~0EW^SDWTj4ncXF=^(V zJZg!V$Jd3yL)1;@f6?c2>Ey@w zIwj>>l=9fePGabHra#4>S%!x1;B+A0*RIPrwSN>qr-tP%mN{~#-5)13r{rjTG^13^ znM{?V#W-wE>yVbS8|Mh2LT;HAftE5+BwGf6JGc_nvArhOWp?top(y#KJ{l*nlT^m3 zpGfivf*GihcO2;~^WUo#X$sd+FVUGFmbSb{TKR}!&M^L%@Ltc_T7Xh_xO1WsvL4** zti+`wg>`A^`GEqdOe)T%bAa6F3~qw_zGwL~DJN`LSXv61{BF?}R;oRs2Fbd%mYjj7 z!1<@}v4Ut7OzAaw;)ZJp?1O~s=Wn?lwQ8nbg`A{#+N}T$h8g`J*02a7Zhie(wCiuy z?z~urlhu@BJ2C~YUa{zQwuL@dPS_)C1@CjVJwk`65_JHP!U9)XBRcdnZL;tgvVU01 zk%2h?8!8hnhTTKVDy)ocB+9APVJoHU=Bb^^#e)D$V)=MmBzP~m$y{Q>X#S!jlGb=t zK^4rA0UfLSO$z;Q%Q7Y!he8eRHdWfa)P*N;UE(-aydCj}F!g=FSHjrb^x&dg<_%vM zb$io~ozF1u3msIK+k2Y)#FCm)j+|e4F_T$FPhK1_M~$16PZ0F2Nm5t(?eR3#Fj|yO zEOBuw!kBQ{H?MLn0yR}88FzdevGEOI@N}QY;qyTEjaw1KVG1t$$$)x{EQu~5=)@N} z57yXjGqDJlWS&K^d^k`dsYe8h`1*2HjINU~?IE|N6fFx1WNb@Kg%GG`U=P720-s%&=m&O7yzZmQv#`>@Za zq1hNmifwqmM9MVm;LYFTP1f-fi+_ltsOyST+VB<&85eP3^`Mynm71B7soP{QwV>PpVg~Y5G7AN?Y9NLB zSCk!io1b77NairwCRTJWslw5CTLAC!+Yr~D&`2?EU3DP6>){PKfX;ixO_1Sq3FCLEJgytZQB2dPk_B2%a*k9TuSVTche74U0(K|{VW zHKNIc55`nV7|~_3+mF9-+r%FKAf&ROl1Vo6leEF;EM??CAdLCC9f#(bPH*VG`4Wfm z`u6iUSCgX`JFSy`yD-7eMvxQc@I8lum*`A7To@mC>_WJGs-efPV!o~NH}>={KWS|} z(WgRfRj3+OOi!1CDG~AXpPMenj)5`Na(K|dSrlj7kv#c>$(?jPyZ89AqH=fT=T0|q zJmChfW*^G_UaE#jPYsC@ri_YzjKucE2y;gMoipwXy2<6DOyoNwX26Hn0d{U0UjlnU zEQscd&2gACjj^FLmBT&@F1V+C_4A)aEeN$du8d?|F}{hks7~H6vuO*E|cm z$9P~v5X$U*#A&eUl_-6oI!rTc$dah6pZHX=tGrKHrh*T&zJTSeQU3Og z3Mt%f7PBKd&X(Rf71pEf`|rx&!*J1JDo;0SMcy&y0c^i32Uf*ti|ck4aboEm`k4WSdv;;db)XyO0Rs4z2 zlBkN2>v<0A@SM#OB%Glk&(uLes-3=~Cu6-3J-~C}MtF+Fbh#3lGIDlOOKh0kt~G=qUw0Q97z1_2Y8Z?L77o_6wJRbqqy zvwN}Nyrrz_lHQ$hEvF1HOXrk)mX3#hnt#Pa>Jh6cE}w_GTez7z)j^@0`ai-kH=NL2HthIvAc$+y$y1jQCDKa&C5w z^B1OdPV-`Dv8@c4#WSv;?SQ#aK5;M_j$IJ!*Jyi*fQrz~Togs@WZ4xN@+;caiR}sv zq*S6ZD%8@RF8Mw|u8Xm;4x)FHdS4|4yQU36Ud zPq<)>Nk**6NIDe4sO)U|9n^(F_c9p0_@!@jDUJeLEfE43s1ul>R4#-%JD1{2vgNrd zWi5~0ETu~WcwcCdYe{etz7%~v^tgBYti3)xQxF2xx|4dU_(7lpuSm zOFVOtV?IIx6XX#X88y!^Z-za#=hyw6ZAgY(p{=(i;tY<#7Ig@o(y8Z0I%?BVMa%d) z)|6_VTONF9ZY1RzsFdAXBQCeFHJJedjE1&c(N4yXd|;qB89ow+3ZHu?H7~?DXc(j( z8HhrsqH!)5M&u?=C-6=y79LsO(OPI<-6PS7@VcntGu~WHbhbmnE)k=!rtzt+s)pJf zP27B;@sC?4pQYh1Nt>DE=?WSCAQyMy9vfu_Pv#)N4^sPW=yIMRpL|tU!^AUGgj7@} z>c%a04!UtvVex{}$z%b9Fvg;QM9r9UwSB2DS!k(0P6pQ4`pjGvm^+rB+1wl3?DB(g zWM|420ijV$cxoe%@n?Ma5q$l2E+vhc2*#CwYYy~F9r^ebbie6m?SS#*@*hiKtZ0zp zA;n2bQV#E_e~tbsgPzd}$Is2}ytsgR)TnWgeMFDj5^*G_C60XRuV#>)v?8@Jm;I_1FECNiia@ME8Hn% z6>0aRyhwc-jvwo?fUf#M-WO3NRYs=qiS-!U9nBdX@aa31h<-`KqgK{Dd*P~k#yX)3 zAr;Ki%x+{++$^xuXZyVa!)x3<+ITmZ@D11Zah0Vd2g@0+ax3Cq3-8jSM<*(VJb4)% z06O{3Tui0w(W4k)OC=}2op~8~>&lMpwd`vZYyv+(^q9%Kco_xL45CMgDP{6&>n7Gp zR>$+$qV#u4877!X`=~2$hj?xKa}eA0{O_fHpCUiI!LH*3;S82tZ#b%Fq3AaGE2ky} zEX(0ypjBNr?Y=V{78+U7gS8tj1fGitN&8BLzJRt(UXqjz#(NbGnp}0hPP{MclLFI> z&(={nuLgSuSZ&{w>2B!M5GC7zT2S~&6Uz0Y`XZ`U=Kr|m)Ur5(Hk}Pt?dkpNG$TSs z){5I%^0v;UaAP7fdV8Z$u5tKV6gmReZy8U~xmOMv1rO{L%+K7Im+j-56+=(hb_@l~ z%14D9@@Hl6LsSXZ|NKx0os8CMb1@PgY}^9C{I30UO5_uUqF?=mSUTV=J{lF!8MoLl>WvyaWAa>W_1%55V5dB5=B@D^@rB1)fsbU=BT8TVx)+X2MEAYl59g z^$+FsdpGMiWqg9s#LCZgM;!^C^6spIJ%!fbbDZ5=}g*tn}6xp`Pl8(5+F+ zNO?NxZU?{SNLju+2zXHroTBYBGES@Z0)VqVKNk`?8xw58fdXBgxPNHZIN-uC;cHiS z8O6z34bYY+b4S<@t*+OB8q#v~lSf0j+I}#e$lThQG5g=+oeeSCK0cG;`ynTqo!pY* zH!u9B^^Kh?sx67275DzvRBsGqKx~ zBj$(mvD=%X6u+}WT4BtQ!7jwhigxE#Cl+LFHT+(ks@JHUG_dhYXbDZK74PuK#jZBS z?i%?F&>{aSwP7hB4z_9^j;_~@(~&$4i9CUK7ev365!xzIjRjw@uyLOGJahSlrOs87qdntc^S?t~$9#w8^@*A0(bQ@^Ns<)BI3$Ik*R z?{w|SF4=tTfM&(2)yF@Ymp=KvQgyG6&U*epUVQj@BG&oG>B18|f|QK$L z^|Oc|tF=-q4W%h+5Bc~Dz=VLWG%kSq=J$=eEtm zT~pBaEipHC^3%azN2{)Y1M`V74CtGO@EpujtgZEQd_}bDF_&@w+b@DO6-!yT{XH_&gsCjA zRWD&US9${lE(S`q#j$h zAR#CVuCR_$>>bl$sm$!*T+IN%6=U#zLl^! zXnfrzBgA53h08F(>OpjbWDlM%i?tJzb4Ynl5$?OqptS%Mq^fIYvF-zG!?o>q8|QN@ zoG9;DZ+|mru~JIJjO}fC-6jU2+-+j0vbIbkD$@eiH0xA|l}Vrv z)A5`c-Rhhyup`s@`Y@WrkJ^3lo1~bE&A@t2@yZ?CB+iHG4~2@*-xD*z5IAd6-(qycdC9c(C4cDa@&m+ z3>CMQDKnVIk_OnOS%&U>x>M_v&t=sv2O$WQYC@KE>22_fy0!9x4zj~lHTyLm3;F1_ zdl)C4t<``W`WsY0p)0|E z-JhXOha0E0(?0FCUur3AHTs-JUun}=Tw9s56t$^$c-mx1Za*V{$@!YQACD&t;jB>_ zKPr=HyZeC}mT_O*H&8J?E=J^>=gJ(!|F{&oZyz{EmV_Zf}g1(n?y1 zSH%=&WT;p-KrlqL5f&HeIO{ey0lIpqF)7rYu)PcX_&C1)b=l@Zd*2B;U4)){0hO10 z_rrF0>-~@ZwulkEt!(wV3H(pl-EGeVtA+!R@BG=_kHkA|zmme#istRE4(crgXbX?JDAO!Es-J6uHOh@G{UaQnW?^n}W#;!VTv)S%J^Zv@I$7 z_9bqQJo@Pp?ST&G@UN`E!r)z&E#yaF3vm2u%WZ#|-F9N*zXW@~o!zR00Xh(wV(9r~ zgK-L6m*7H(-au4p$-Qy5af;$L&-E?Z@N>W(dt~HgrrSWG0C5$7FFOp34_6j*Ge@jEn)x zzmKlqme`_3>SiX~2?+2LIuv9qDJ_260ctlzKlL#}Zc@WAKU;VW%mgrMSlS6qQA}k( zOseP8gqdWaGU~*+RyY09!A4~gx-^Q!4^YL$GA&j(m&p1A%~o0!c_ExiwM|<$)1-F7 zP^Tc+RH7C`{ZmY1Be)jB3^0uWO$)N&!(5Z9Yx0bX>z_Xl?!Rf~ytaAQ{ES0Zg5|FEBVS7AT)Pe%9KT8ZM&>Fdh4&&9r1R8`S|)cHc-QNK`z&CYN?fXafrmjNxj#*S6~Y(iVkI;pI2C)}C&tgZ z3wAuz!;W~ z3+OvwP-MwvkhM?nPiEZSn-nsG(OmUJl%!KGQ*T9Fz$^@nW(e^IJK8*TP5S~eKZWx5 zp8C?!4!P08)-DIJLpgts(=0BrTiIKx=kWBk(C^O0LOx1I(^JnHj}|Deq00EdNHB_5 zb`+8Zd{Qlrbw!A`xocQ6fgd+|`WTO&2VW>sy5jb50v8 zgZe?}f607M?t(tIVk5GKr*7V3&XcqJ5w!uH1oumrV(s+wgmn(#q$3W%S%P9D;Da=; zo=QtZMJ09QZt~{DGWcts{%`CRy556itSQyj2RFFcHXGGfvR1A@Q8$cqd87JA7g*14 zBNK~`xNk4GBX|CowbhM=J()eiQ)nWZCK$p@oX(}k3)Sp39>)%+{7%WRE(%tc`lX6V z8qG2`_{{?9U90LMWS+&>HhYFuFN>>(CRUpaU)K%tq5ObN7Zy|c-t``@Mx01J)d8!m zxoQX(t-%}$!_|>{ZjajtD=b6+72tq>KWInJ1AEdbH9juP8Xta=>0{CUoGK(QmQL8U z(_x&~s(SQo#r7JLVbI=n_oa61ZYAl4n{6=) za#T849F#R)p4tN?eKlT$c7E}5reb6Qt`g|fG0RIZ38bJ{W^QG*u>SkGd9RO^7F9XnTEqz+&_aa(&y=VLVL@6qHYApcMxZ+kMUCa5%y%W)=N@HoCi zB5VpY#|5%On)gYCIQQeki_}p;v|6Rbc0i5(GY&p(ASx5LRepd1w=ept9voYB;Rzs8 zqCPyd-OeuDK2|P5rP?IbFBv0DaGLy#o$U*+=p5e@@$8NjCE;B03M z`cI_m4W;^@+`n;F|A9ljF-;I(62>m({}5PgTF&MsuBKMzU}a4RuVoKGfvoKK{Ad86t6m!$kR?)I zxPZ;=Kh#QR(AS-mmMP1baiwDncKaQfb4H| zWz&7b=k$5l-vB&z2r$RN$_eD>F$8b{AyPhmRz7a7x43!u`B))P9S;Q3<6`CHJUUBwRI=3{5&=ivqLKxQ5wklPRd zp}Lgha&83VhSyfdt9V!^HvM z;f5^82jqpM7GlK%d<%nzog3m35-2A>H!D9U1U=;AVrOON<@*nH9xfon69+`{?`0u7 z=i_4q@z=B8h+CN)||Z__-h;Bnu~yla-U5 z7r?>J$H~gh#lZ#O=jLSPHiO7$idFT@jpX`r(Pc3 zTGGD`N?Y%4J36R43KGGH^7h3O&``Bl31a>s)YR0pK|fsm6^Uv()v zS}oJGEZ3}j2lY<40tSk6d3nvQPv%w{N{S2`Gw0*4A9K&44~3TRuGrpLIT$SeC{y;7 zx_fztVK*9$U(~D-8daL?G9;xitNc^W?I-A%eG1X~`ex#&1;(McJA@#nA4@k6 zt_z5AqOt5n9wSH6>1#_4?-oy7j;h?_fsXh@ zVb|jwZgyeljULLM%9FBSsCqD5YstKI{84uSP;l4dtoZe?dhtH32uynl?K~9K@SSFbEVrhgy zL_wT`Q|r>6Oc|lZpkkX4$;3sJyv4xhYVl-u_)D!I&>~}lM6#B@x2bW?1<)+w?z11k zttgxh1D+H1RkhS=fIFQKz&q$K)(<(t;XCFa>-DefHQ2@|5?tJyo}q{8exh6a%`Xo9 z%&)N~c%_)K+g^s#R8QEn6s-8R#_yjIW+`tM#4%Wc86Q|aNU_Xb8<+1H9~eKI@b8}+ zpCd{dm$Q@6B2j%NiZP^mLPMR!_>99q5Oj{U?yqIQOSs_0<-aM?=>QMfXx^XBZhlfK z%7J1vJuU3sWeP^BpL-?gWrC?SM5kZ)jG1lLNLflhHimc{j9`qn-y{>E930KFBgd9&sT5tZYQ4HB=1+B$OG_@J_{Xmnk$k9DhkxS6XtO5OkL3hVo zue*ico>$MEvP4B!#PP26ndAbM5G|=}TW)L!c;JT04gk>+r1P0x(hR#=t5NB*SR>~ zS1?x-*0pd{eq4t{H>hVv=JVwxYOj7t+bNWAqBE!quRk{2*q7W8F@4-&#!ZHc+}t-{ zTb2Vos$}TW;N?q_DZ^3n>G3DN*dL^J4XSlytB60bsy1&W);QNLeJp#3$chh|ZkQ>o zFsLHw_I@^p$_Co1m~;;LFb1W+ni~9Wd$nlY^!(h3-)s3fqaMm5&09%y96^%6G;v^v z=5MaQk;}bA+F{H+Ris{22Ed*q4#_la2&+%DmzxaZpqfJHdoh%cIN20S`VJlkw!KO1 z?D-}O!$g-M_Wp*7-;G1+C)ve7N4%B$m#VLsrqcA1A7GVd`H~tC{v={_Y;`Q|LrHsb z#@Y4<{s>$7cIien9%8(epN0*TOrV7yRdaw>765Orvek*KY9?O@?R822Alz?L1c^Ty z=eSdW@C-3SZ<$TEds+UnUWK{z4BYSaoo=~@;j;Q`yAv_i7}BnzFXUIJY_^;CPVl** ztVZ46E6fDcJ|o%Ben=mGLHJ7)MLwiQUR9^+C_NzZ( zN{&6pN&SLtj0Mdi-Au?~NbsvejWzHGuWbsdgGRQXvtzx2T-2zA-fd%LaJN7iIz1}| z6%t|JSjkX6^Y19yViNPcN*w}b)7{axK5j|8!ZiYOigAjjM@rg8UZxI_Uv0m&eNHPv z_J|K1b4TBfu8eHmx(YGby;^J)`V_#2lPcN+*W-RDw&cVzE}CQND;oS)zJH4Hg=@={ zV9NHxdi?rVpDn>=yzJzsc9Z?Wl*j(l+%%Ffai(Re*ID;=W>yE(K~raTGA_2*zA+9# zhx_zU{>WXx1O0=G8ve=nXOEm5;V(db0epA4OGt72O?(=hNUE2P{%c$1%fC-06%jav_Hj?TiWxIxlU*3rQ*R_B2g zhry3z{(5cj&?7FM#9f;=e~4zhDU-PL-Z(lcqHzEH|h zRVNf45ZHpx(^SveX)`iwKDMG$?*{%-dGe{@;fTk!y5+Xjbz#GC`ys=3zHW0bx@pyO zrD#ImzH58Z=+)x5vF+^TRQT@Ytn-oF-SJTpL-gt(CM#C>r9V+s_&Lfl^=@gcYoo5o zZNtUS{U!LBt@GxJ{PkwE908ro^Yr4_fyCpwvT5V1+lJR`AV#e3&1H?M$n(kdHr=$2 zty8gXAolTKjU!7-%&Pd)4ZtVbm(hR;gkz6OhO>@?w#Dt#`GIbo%un-aLpzbEbg(?! zruJK|MY-ubv}?e}f=|UF6=!NW^@_KMUP;0;*&uRrqc}Nip722_MH92FxcMj#B6}QkHh`QyAJ!^=qAaGTzg?0O}3q`!{h1Z zurG7iUY<^|uP#k$&waYhT{+AL->G`KeOl*Ycb0wUYhgxeI2BH}N$~-LH-6>^?qqlMXY5tz zuE3$+#(W6G;n2LjS&n#}fh)hse|$ao&Gi@WEO2P^{6cs=wy{t1f`}xj*qDO57tbnS zp$9GWH+wsrLpPQ?c4v_E1#3Inc{kTI>TAf#CX74AdMMssVsA9wJ`=0p^3M#+U(GMx zF^A$#!=Fw=8BQZH1P(*yiGs1s(tX*gjx@Au2&GBpZ_u8`-#)ho6us(W63*~vr{>IkVs-z_TOd&!Wv4}Vd- z#X}PKqVNEh)L-8#l!{3+s-q=`E1dlPYDgW=S0d4{gPb6hAdft+E=&>BEB}{Co~Ccu zF+m26J1RtzibtKRVH%M$Ac{I)PM~2{jz~JcGJOT92AorN>$THl`Pe4?gF! zy&yVy&%POSx?HB&`*bB_V>$-P0W)LX4qZP= z|B}lSLyDsRqExCYJo1N(V@ds^Ktcm4+Tsj*N&b4i$wHE%hmei1r6$`hm~uaFj4umH zW&a%=M=6{dyK~UT6E$Jk2of8%biuXY8$0P3qWPaghsY|CBkforXlU3u5~Q_50oL^>;*%SA(+B=*J%Svf{~ zt1B&KH(dOIbR%F)0TeneYJ;}J#T)eONzc~V&iw=0DV&zu zVFuQ+a&umIcJxguX&aC~=`WdFSLm0-kbGU<_mlq{x?LtKS7o%g*y|YRm7iZi|Gg=l zHMR~B$x_m>dZ3MSHohyDjCW>qj(u%c-mTuFB4z{%{zNSu?W1M$U#6Y~pxmvgz^D)++EBVO%;*Gz_L=$m;Y2m^m7tdv zIoE+@;YlJbqv*cfDq|hzG@-tjZRanYlzu38UkKg8(oYRkG_>%eac$BXen@5oU<~a* zld$KuxF_m?oxdLS8sM4qX$O!*wVIKE5aojlL2{p~@lpntZBM>V`B7+q)Myz-@=?Ey z1(CvV!u`6Jo`GIf`fhAf(`FUtpQ^+N2ZuU&d+n@I6#2XNZ%FUB9ud*|1a{||*e&k% zNWGaN&+iJZ{HrlU2oG^}qXV~t;E7Xxkv?@1pQ=cGL4rhLT-p1m=!%7S`~zs$Z7q!N zg8UqgSIqe^9h&UdpjEzsT$QB{T{XE2UG}+>Jp8~jMz)Pb9fPS9*IZ<;=c%>#$LY?t z(L-7r5`(q(q?#E(0gGm2lNL=KG}!V`yh$(|+$f3wJ$W}Or$=faD6Ml$9yp$MylmO) zY}==lijPs*`!K@OR@Ke1AcD)qd0q(9QBhcnmxQf^@=0iHQSC&8u3JEa@d|z2{)DFc zF-eX5>)K5yFqe^c5Fa^*tcpi-DnYv*62@(UTrn?&@qLfdBWF+{q9SIM z#D}Qqi0Odos2_+u(bGZGVLvc)p>oM`(T$-bSbDvKJP=*5Y%mQFT`(=E#wF6i*)F^edZD3bt6;`B1~5(hLb>$}K#9P8$&yY8u++{1PB}DcYJsDCUn%=z9c1bKTEDAzSTr1Vl$!BWe|H@RE}4 z^QZ&v#aw3BIvhBsS%YdxO$oVj4sen>=gFwj?qh_t?qdX;Ud5bdS1FukH!18mXIZ^! zX-#Pqfj^B37WMbdXqzD$RjQs;O4CS6SBqk=>l2Ru8fzq15^bij4BJYR{shfwHEuf{CB!IWOa2W0ybA z^HW+V*b6H9isirNs;%{@>a`OZC#X$bp7$Cjph1*>_gCi{!+VROlqZ$7GmdI{QmQ_w$8S4Gb!p4FPxigiBn~27U@UX%G=;D#z0mFzyut?%k zn3(j?;*M2t-egS-x2hEp&w-8uFTQCeepBBUH|CjcMUQbF1wRhFH0CXCB;Y<~5DoZB zX`DEOMLfHT*N_)QJoDr&Vth4m3&N)(ia2dG(PmJY#u7qucsN+OaGHO8;w(H*Nrg|lnc~lt&fyypFjOKdp?f8g=y%w#^zVxXMnZ4IN3(wlT z0!5cI+?i28Ikn||c2;hQUe;QCtU1SAjc_aR`tfRl z@!Rm!Y2Ch;cn)p22G>L4M)*T0#!)e$8S1(%87+A?j!oa=5II!b3-MEQyJS`0iUjOB z-N#|X=IFgRBI7)2iVotGO?{MM;vKY0`t_}pO=7}RI{sm zxKcFYQ1w<&h&!7$DD?QW&YYvgdEu3DRn@51rkk37{rfKS1?ZELyW`jSdi|Vp{>o;Q z!fi~d2dKct_kYJpEtp^66}%CCH;ReYvIuw80h{se@cG$?x}yqD-9jd4$zG*4)Y87AsR(7H1!fHt{q0$e3nr$lEOGYeMJOx&`;W;B?AQir>5poUU8!NrVm(M)0hD1Y`R$F=CZTX`Qyu_ zqVN#LngYxpq^5{SGOCKhGH|lyXr>0nJWi_NcrGYa_F*5d^yfZ0t z?tazMWymRlki#B9cj0*y@Xoz zpfF$ofioH^qPnh*pt{6 z1g!svJ^^@*h+k3gD|3sf5F@I0!xM{r{&`0*P3ZaTAei3*tc0iQ3vo0q(-!10u*Fr& z@hKAd8PEMxZn`{NR?cIIfYawHsYG5-_-8Y}>Q+q6IGxa2Oyvu;M%{cuRx)vAP5Ecw zS7AjGqYtlpV@AG@y%}Sv!l#deZoN&RbBPBIg~T~&TPvr6Zj*P*q;8_pXH;Br43~Ku zoBYh{!93rf@fOdN8iTR$D|&qgntSA;2UxBc(>Zx=hpDw`YrmEvWE~<{QftOnFeS)y z%TquT&n6{)diAixBc{LtYDQWo%co)gB9$>t>9ITDM0A;ul8Z--=Els*O;W3D(V~tR zULBoFO^)v7op|{>X#2R)hhTeq8NIN4y2n(z^eLR5QEG zdb-xz7|xv)$vVf|*hI0UK70}(C6ony*vwBo=^Og{mMfe|ZrIHXXFEGYox zqu9@YHp>{w(E|p8^dLKwkXn{pirDkLFDHMx4(Dte1T(4l`SQY6rbX$@iNjVnhVbO~C=JwhhT*7_-EXdbpo!;H z&Q0U?d={}p){jm3F!XVh*@Zby;H1m*wka9GQFO*nuy2Ywt_=yUY zthSM^$=dbbT;!3`CBMn@#(pN0)40*9MTQ7TtH2?A4Qg!h08)`>TN?=sjfaVqplvWGZW9}VzVAIVwh31*5zz_ zmoDsAO`Q60ktA){HzufSXL$O!VClzrLYmT6RddpMwaUQuN5~tz{L{%)V1IWaCoa|K z>c}uQQ^HVhdeSYAe1v^HJt2UC;r+b|<+^-?20hehRx&N8sb-57bBC3fP;i>dZ5C%% z8QRD3Y}42RRkEw-_tmKdF8egrclI|0jIRs5@->(y6yen8fz+?Olb5R=lzCM2);ew4 z3YyV{GRR)h2+`z3s>IUVx&`W^4atjpi{I?l8p&3G86d^7X>Cv#b^Q2qLbQysN5R(# z>~xdr@kB&9p+vcMRRuC8ObVo|SDO7Bb~Cj8Ek+@nQWguV1{I`gohYDb83%^$7s7MS zGwxd=48mBVDFU?5^+^L$(Zv$pe^d%}b6JDCO?GX9k!eOr;IIiZcql0)G^^<0WqEkC z^i6KG9O9SMLu4#Evvly~1P>;PRMi!W?q>cLsoMUGfyHJ&maNaKJ{PMz#1-m*W^+W4 zQ=H{{KXah_eMUJ(ir*adn<(QfC^|+-2@hJupZ{An=yj#xQEs&uPB;7Vmu|F&(`4%T zMVHN1Hl{#Vn4{`S#WXZBG3b-+C+xE0=+qP}n)7G@7ZQEUQNOs^eI0ea|^ZSQ9}SZ=0*;KKxHHdoaDPP3gsa?lzM+U^E0WLI`iXUIr{{7dEuXLqQ&IB?Ei?x$r%W zA?J`NltYF;QG51Ac%;ih9yHuE9<<=KxHmL*R)WaTc77y<|7$1*3vZx~^(piZuUps1Be zayHYZGhO}7F=uq`(X>_Hb`>wBEzDkAW4d4GP)SYP-B%*FPdSZR?cPWp+PsU}=b z^g|mbXWl(k534WJO(*Oq08C&>6C@|p)LD?cOzRats@|I4GJP~YG!~{fkq!HI!T0G* zONfiUdrFnV3Y^F+0YYRL8_A6hk|Wu98r=+k%iaT(HO*5Rki4! zP!WqX0X<+ta-CFIKPYk|Q$&t|PM#leI@>pI?sPY_x`%gk9iZe9L#ou`)y$+urTon> zW~S1GIgJ12eFvyNcGKq*29FcH&)mqGg84cEGrE`@^Ionay z+MvQ^F`#orV5VCns7__@t;$X`cgqmzfy(I2C!_!^zzfoM(k>dlIqN67l;i~Ce!Vkz zEP4W^waM$zFB8VIo{$623<51JRwKRFsxbT4^&oA%_fxpVm+EFax8K=v^jTs>hwIg? z8RDhA_8SKjD-dk@T@OPdGg`XK%ZiMN{rKtQv3^)kFqw|J=)@uid7v$TKs5rpxO=4# zoxB=KRz!xomo8`cZPkXPvfkXP0aDw-c_i>GMHJw?JuO5o4+m|qV$fpH-w+-%qyY?( z6EQAZihD!D2j=9wJZ#nz(8s&6kDK0?+v4VPl3)w>7fa0$bd`u~{0@5+9&UWk2QXoM zazj5rI4*G&v3DaaBQ()Sgrv0Wu#4+=P|JwU>SGhB8<>tk=X9%b+BLE=tb5z%k!TRV=qB2^5?h_y zoXQN8oRz~wD3*wwD^N1EvWj^OGwOwxDr; zuwdp3m?}ubLrz)a^9o8nf!gpo7GuHv5Uk%olc)MA&N|H?=(vB`z10TWcD>wZGfB4pE<2wX&ysD*C%8B0ctv_q{02CSmz^f3 zk@I@z=j!!{)S$ui?Z7zO1j~Ob~fXHoMwZDnCxy7@>NF!*?tsai;9U zeNdMBWAKCPK;C1oou&iCiy>&VLYL(3tOm4UIuO zSeCt8OJ;y8F;rEHAa@o5UK+Lb69_t-Gl_xZTX`@eh*#UX%m{N-6f<*Vl+z{YY>|2- zvj%g+kq$7|%d(w}Xe zW4vj`Z;_Hq%pO4M07`N9@(NOyo~yFAqpNP%;o*C-ir0SklDT$MUnayRo}oNeGESVD z);j12u^pDE8I7OL#Nu8mU_*lGTuO=Fjcy7MgPN}kJt_T`mCt-?v`@K=%pgdpna;=C zbAf9KDr(NppYp2FKabIFmzx36@s{5v-tY&ZwOz%%(nEFb<09mFaqvQ)+i{ts(*g&dQgJvOxk7#gr3AzMfK_MNVk zZxJ{aSozN@eY84rta_9Np51`Lyk*b6K`A(Kl6_RJ#cy+Xyp~UTpWJbc{rb6wO zuabNUzoWD1M@XX4%fPD-^Q*RSQZW4f?CvmN7OPI3n3O&}mAZhnB^_C|L2RzJjgvln zDUCl2Lo{>*e+iu1fF3t^HWy0wS`^5RYRjBogcdzqTRQl_A<$iB#IF0oA&<&j2@$<&Lh%{^;iGccjIsaG!^)bDAvUA zk6mTH&5&LH`(7es_gS{hVVsLRVL#HePUB9PPQT4p6I9^MnF4>pjE&u-F8fXnkVm^d z?&Wp4*<%i5Rn7d@l$hkB#^ciJMgK^9+=!aYYa3~d|7el;a@;mUbK#(5?v3>HH#O?(%)(l!qijc)n7Kc+v{!BRO%25aH8 zLDd5&AEX7g$Q9cCKNH&IU*4{Kr+F2VCv@r^4m;;29QoSrs;wg~3okPWd}_=gNnV=z z%o1p8_;WDc`h(|VwiONmROA^{^DaZA_^_W0nW5GIwDd^ypHq6D>}^4UUp>!%3g$RC zHZ25ZrmceV<6T3y^1Ml@F?DICuiua;MTK_6XGu`VTbs@|H=Rh^^fZzl)}r6L&}MTy zrjeV|reG)&@jmv>RFG|;RN!ObOUh!D2EoICjNwQrVj#oD`e;c{%FB{93q>Zx%>2=t7>kJ;17%KvI0z39gtUS!NWfLI?S|3*Z8JTZ{-*}(C>o+Rv(mJm zjJsh5dJ1DXI!hbXiljDwY?yVG#)4W1Sm{%(;FNGPt%Bau%$%kqC2&D6y;!sq4l8|ozh*h{0sx}; zla{B19}=CLKzOXB{Gnak(X(|vRev%agFKMIr_~)kwB%ybnjtoB<4%8OW#i~xj>cTRKCUZ`y+Yr~%fPX^z7e2YnpRL>56(qxCO}3L=B5D+ z_vWvoA6*qvn;sm1gKqpP9UQgGoQrbJS!9&3V6*9CM>I@L3 zW-*RA`i7>IhZIUQ5$y98BUlb1juUUr=|V`eqZ8tj#P(tHCtIyBxtkIYB zh4yyW%H-rQE689*^P7$MX!G0mSuQ?v<4vYLmtAI-Nv1cpD@_$XFU1rR4}@89P$8J= zx?ihd_jOI&9u+~#cGcaY{A~}!bXAd6;JBLO&p6MKcAa|M1cqw*;g__6MbEoVe@#Q`=$TO{jCizAEXTozm0 z&>C%Nx79oE7t(JOTm9elJE$e-ciPVS++Kg&jX~S-;ZLbh+*G^~xG|Ka+)nGy#A2@* zODy6EMuWNi;aG{&3btpx!8hA45~OaJ2SvBoX@Jw~2#}rvQ{Q!mI<((DZF0^I%{|n= zNF)hNcY@%hDjY`RNX&SZ9wJ{~gHictkE|KnknMX;zU45gs5}6S&~9R%SwDgKOYDS5Gv@izZcm%UME#o|GBongQ7z~c3>+;ZPfj7TaWQ2A(uqgVNwH zIDyWM@E#@tm*;}-u2&Y8@G-2g8tukY?aDzFO}7@8dwi*o>U#Op`sqo}Lex3ODOIcJ zd$N6Lnl7$JlNu6(_UxZAJ)HE9{faub((x3CV55$qZgyWjuLOZB5!-_$WQF1hYrk`E z%t_zr`!yxvZof>O4h_=aE;B(3PID~lb%PDE6X3R^q3HvII%>K9f3op^@#_B=(*F%s zeo@vhdHP?m8+8>&t$G;{+fS(o8X%o7Wg@G>k%8uK>|9Qz0U8n*$G7i8|{Om zq}$H24(NB=ErMCLg3GrH{z$sx%9d0QjVx6`k0ffN36p*F!zk%`eXOU&Wwo4f$nB04 zupm&%L(T`|LuNtF1Dh67?_7}wf1S2@-)ms3PBQe%3+wb&+GaE2wW_&!#Sj@ux&MD0?Z0IC{}8+X zcC!EFcE1SpS2m^pj_Lm)?EhbhHaqLTSv3o0=-mBz@y^w(gE ze}V2VMbGfX=-Ix^IRgVT=l>_P1a*nT7^riCuJ1=KvWToTyg8u)TGXG-z zbgZll9E7yYOkalniH&mR8m%vo62|B~+O(* zk)7@Tx*?u*c={;MKd`qyILICBk&e%?*(MZA=9$`zDwMLof+cy{#Y?4qX`yfCR2W!a zlmSxiM}Gc~U?vSjFeD1pZs1>n0$}|Wkm^WDjl&HOl97MzA79rGJvR^Eci-cBA1iqg ztZRNhtQ-~sLjqyuf?%>_aCq+Z5iH8TfajLs2HPF<615*I)N_ppO3smJ2`AFg}nWFQZ`Enw<%dfg4>HGXv?*Pelikc)|* z(Ka_@_AcQAa9v+{o09OP7J%+vp>o#)R>@s%_s-E_PkiFv=MAtQ@H;qf^#kS z#kOLHIdABhkN&5vSQh}{qOvs;YLPud9`E-|!iq?x1lo?)?GX1P5JWZ)6o~!gG)`Sb zP9+a41u0MePn8ce2O_)W%od#`(62a7hVV|1N0?d`$XizitkKTE>j#OZs8)?m{+)06 zh>Sq-2@dS-81LltHe^BQ<}RWc;yj!;DuNB%~QSp?ibs^cqAoN;H9n1Bp#!?8DE;q{rB! zI>2b3e=RL?q$t_kn)TV76=2LvgjZ{jLs>%f;ojhnkzMv@_xcnvpuFag%pheU--+3n zqaR=xcF6pk>~nyQ5f-x&d_sad;-buTTM7VBP52x5Y=l1=08(%FIk%rsj9=XQ2AA%o z407xVK>&nq*ku=yb-Gb$i(BN$5NHyEL`p~<$9$NRZNP$lv?+3+C<4Q%K3p-)0kJ>u zF)27EI%mv0l=_g?Rb?XPuS*Qc+JLGUFZDdBCXBz`v~lhbvlQ_Q8goLkcP2uuAExq#Y26^H7fJ!n7)(?r;V(;S3`Aj*@1kE%IU9hUy^yKIIr=b9@Qq@y5`+^NO0rf;3VLW|xWlT6Zy$vT` z1FmMO`Lxrq`g#z%$X8KpA)SzE{dG2#_S#3k;?lgzM;y;H@N!8p%Z4kX^+La%S9B3OgTm6&CEpAD8F-;rJ{+y(KgP33>m zcZJfSOyQCm7wK|L{vMhPx`MrNLFSiWei9^>v^M6K5a5ejn1PtT>YBC|bnCzUhStn< z>CgWD`Ry(afNp!Rvcz8$REIfnsF;V6P9Vt~T$63&g_aKzUo}oN3fPtSUCd9(bYU9~ z#RF?w$N<=BbMVErI8?-7=;*p*Ju`vXGW26`OH~m7>C}ipD?FacHgT7*;zv?e{QtcSj?#P z*^KG)8<3s=SB)NDP5i9m;!w%8R!e(wxi`$pqZe<#BaaThzac?LGzy#25~rd0$*8_` zO{u=5z91*Mn+E}`)RC(g2>UK(m*f%OilNSBKwat@=R-m}XJhEHk@cPqtbatD5&d@i zWoLUya$kPLJHYT=4_h8W_?_?je)m%_uXfc+hW?hLD*;|^j68+?k7FgcYVH*Q*LG?Y zuJE@$$JpKdp<4<=z6p}FaY5fcmp+4T6jxfHH6wjQZ({90A9U|&kksLr{4T_q=yoecupb2?0-1*tLNpW>ArZr@V@@BkEuJWhYCQ4WFF zeYJ8P$X@Ako$%J(DS9D2i_jo&(?*D7r1L7J)8OvptwF0K3@hI|(O@4aJ%fyT(3jGW zzSvr!69iDjFg45X21#oKt`or4wb2;0dj5>)l_~ibi_%-@b2Tj%83*~}z^We*{Z>m_ znUm`li%69qo&fYUC9oTZOs+>c9y{Yh$aljU0g0jT_HXC4R}{$tWRn3#p13rB0AR=~ zl4tJt#k|2UtFW$*hLxAHOSXSH2?ksgnHUM#-6y@|Y@SI$rGepn^=w!hL_7h8+Vx-L1CI$|5 zi(Sb0_Xe86pjNE)fJ__lrriK*|8?Y*>TZ9X9=`cI=BJs@36E%-hWxK>dEjhKqd~-O zj_8MC7z0`(GF}ioP50)oN(59-srf_ zh$Y0>g3ASv7-BQSRePd!(g`pS04M+zbv9d+M%YIq$DK?>-#^8VtWc$ys}t<|x1kV? ztUtf^<0onBx{}|(_b5P29qWGZ;DK8fZ@-utdT}*+%HqBb?`s>JeX?YKLYeThD0^ax zKaFrM)Ica5`u{LiSIEhY=-B?8e)^FAamtOSSbXzn?m%iXBm*x9J_9TPQCsHF1T=tS z+z!aj`6l1>ec_L>)yw{UR7pYQRy?v@Gu!*h)9=$4{{CZfk8P>B(QZ>u4^({QEl~}8 zM_Zj&U>;J!t{~6LH}A}@JAMO1lLm@;#(?j}i+%gq&E~`wP*-NJm5Hj>1#uG8s%8|y}!)WjQ&0i+J(7|MGA7LdBRL5I*$VQ z@^;W{LU}tTjXlPUzV0{Il9kj@-KNOB72&-0>)k%uetbfea(g3P5^cP9t*6v7^@vgS z$l(g}7ec-d1s&zAfiMG~OeN2;iMEHkl@iR*Gr+lDS_lV2^{UC>gxq2M`0Cr?!J0)F zKq}xtQzL&HF_BqKFRhRTEVAo$9r*r`rc_K}>pNX+55$bx_9w#sbEPh^Dbxx(`iXMN z8|>oF(`5%aP(tKp@FxCqGlsJJ%BT>&Fd9)35@$jGJK{ICukhO-z>oJ6!s}eK!0-K&!mFP-$l4+k!NcqY{JnE1$vo%t;-|8_BX5Cv z;A8B3p1dWhAK=O~6|8nqd7B&NQ}t=0$6vG+`dCL(X>e9Q^R~ZP zU|mVYLnQ=Am*A(Ngg%JQq7qL5TC89mE008RqUfGteWb_`BXU8}9q$Y9dnD38Y7E#? zP-VindBxJ;+K~56w-2QMPyX(l?a5o&H&?&ZPeR+ojpPq1-MDC{^0TDXA>Td68xPNf zw_F}=J@T4l{bcqbf?iAFu)j)Mx99Ym0SdU(@ASd99_a@k`Ux8w$zQxGSqm zFH$UQE7{Odf_e~^jBnUWcwy`|$h0GZ_6`5EZkLFa7e=wN!4G@7{@mHCRyxp=^IC>A=9 zP4xY6$Wu@Ft1#B8c<1f(7}hLn@VPZ@WzB}#;>L%CR=1%R=G`46}gMq5VlsCe%QzK=bYqZhG^W}(^3#{Js&Pw z%WZ_zEh1W|%d2JkY@L=-3}iPS?mgJ!Wa)SqjYkv>rr`TuJn{V4rH^c-&>5t_6Cd?y zU%tM&Zr@ECY7pW?^#9#Nk z8!uQIopz7uba3gN-YLL~Y2;7$WZ5nKwD20K{v}PNPi>!H@>qNzI?kM4!YXf>Jx8-h zGwYM9S)*B`8SuJ@;KU6jUt-fdr8{$T@`!rZ#!(%aV?Oc1fdl+G9(FGvef!7>gEW~h zJ{R=3y)O}_n>K1Mx`saJf4|-F%_B#oa?qbW#@Z^>BWQGc?6a}(qigKgP4!pz?G@sf zQB#l76TDfkP%6aapV3-u-C!w`QyCA~iQX;)U>;y+Lw4LC=PR0akggtj2jJ`=*DJnu zm{UVc*sy*EkcS=#4+Op+vD?B=kPqFD4mds_uiHTF&;?uO9^d%FvanJ_SYf=&4ZJ7D zN3ir2oR4&SpZFVTP`mgB1wmZLp70IQ%2=%A50eB!sQC9ipuYz4@R8&cgvE(s$A$iD zY>X2kd^Z#atoRlsZW#R*wNp$c#b-~)3C0a|ivV51<}ksYyi>$p9JZ45iNr6K9$#3> zykATsWg@2Kd0OQKUNfoKrSa*dX z=EM(``Ns*zCF$Vec+w?dri+WPPq@Tvt8oe9)^Ru}PF>}u91H?bRt8eit<(b-tR)27Dx4J$+{7FnN zg!|U4GsJ$sW}B}e+J4a23&KCBYuNop_l>qQqig)}%KfSBjrW7^^T*{r!Au0F{sQmTp8}|nD(bE!xJw>@@ijtoIhd z75bQ<_FUqToluHGr$5+k#0OwSso|!l89-n5rxr;8HW0gERyiT93^#Heor}EIDg7?tFWr4X~MS z&6Qp4!C65IS#>{JDrb7wuv{fP64q0@O(M3z zW_6cZw`)h>KNuX5NG{*vd;(~@=setbz3|`aK1O5yV7$%y8d5pt+w8RQ1_pyc>$f*O zh_7FN7hyV@<8yLso~>KwH19lFpAab`|HLl!P)y&fiJ-&KTwX(%km)vg75YIHi>wwuj^Z%>*jUqrN_E^`vte& zBVNsG%%bn$Y3J@&GXE`J@A+z2)XKWTQlqTksT%Fgty2cO^-7^*LFQQ~T)j~Dm_X>} zp{)FEVKHf&u3Q-Ju$d~2S>kl=q9tp69~RkEDaTaH8U|F7xCf=fZgjLL4+APns9wAR z{zdB?{^w;wr(9WSwIbeNOthu&ty)ay0 zQoivGR?ThLdX=HFm=CXYfi+9Kt%>wPBfkCd6SHFK=qc_!2W;4Pc%XRVCpheZ(O#h7 zaF~(JASmp}>fHI@r4`NNV$fPB@ZS@Vk5q6a;a!kP!fOq#r;y_HI;11Hi(cM~qH#ne z2DbwYksw{dHJs*%rxef0iQRu%%-Q)E_t_X^+Zz|?Cf!z`VX9>t<04@)x;RJ%!_lJo zYB<+!${{Uwl&txS);VeqCUB}B6xWX!J!0mBtEo*~i>f`)wJM^;P@&F5zUUY9SLf)Jo^1Mx?6x_>qA26BqeZe)VU#K>n=8)sB%)RKg12dwi-; zQtDYXq97_-QeUni-HJsm0+$MNyRiWXo91}4k%jpVs~@u10Tk;7u-c|=B)|PErUoUU zKP)5q-3;k^?wr;j@{`g51zMz!eJzZH3Ti7;D{6IUL4mWK6}?xUs#_}_shQZAxHV=! zNeU;P?nJzh@FhH{E6hk4d6j-)8UZ^2vgvuRV#Go$&&2Q))S!XE{ zfX?1o`K03=1}EA*XG%p1ghB#)MG2p{2u|zUWIjh?qE zospYw_H7Ss-_a-SeL3loB1v6o49i%M^2=n#Me}fdT57|8VUqBKf10S4imO-yD3h$9 zuU%s(_t^(7i@dr%`72fMxaqCR_G+vm48>OVbnxuGRo}|Dv8b`&Q^PD4{er$z->dKE zmG*PDQwb{U4T?)o?Xd{1&jZBK=8oN(r6&)}M$xT>)K8-c$61G_%~S zXwKr2l#~V9IgzxVBwS&P)(gT%rqB|7QQSsNfu6DmBj;-K+CC?7_*^tpFCOc z;*f^D&Qi-F*Y+4QE63g+^*aK?I-5Y=ZiYy%0lUA9HdULbbmLcBMF)!RBPb*^%C+1P zek~%1p2jdPqM!mfg;LL0BNg+nHY1AN9~VF*N6B8sZJ1?Dn$D}ImOryWaXgME?`(L& zve!)-52`Kvnculu}KnA7i&^buKvLXvn90qz9Q+>~V>)kC;ER=JM{?|2=E zlpm)bRbP;N)r$yb+*MwHXIu)skK0s_^(^fF`2695RsyqB9!ojW&S0|L(D0^Sk2t&D z>7dGooJL{%RtHHmo5fOK6}%W4-45^l`v+OY7DQ4f~>W; z->a5ca+B($W8dP2(4`RFdWcRdOp6PKgI7xPmOS0SgNmvfo735_gIuqTeZQMhU69Hh z5!z&YWPIU{pb}${3Ti^6wxlS0*;QPmixLt!T211@P$V?+hL0+M@QfVITe9Cvi7`Ya zSVA;T#Uufmz!I&x_@g}IMn$PILH0aEkEkP0w;F8lNId36vGTq2f&03X07=={kY;Z* z979eHgT+|cO_67#Q1&ADfqkZ_ZfCU_*1i|&+aQ=1PXCdu&7Q5zpTNhzPCs2ArwA;O z@i8p!6Y;%Wx3{tgk`bg4*AA|bT9`AuprMdaIL=%Et1N75N%(Zr^~j%5`S)#>{(?P9 z_Igo#BS{M0S>s~>8b?3XrrEsKHK6g{eU_jrLts%)9Aic_E)=GYLl8stJ?)^1e6(Ds z#(R}}2qkVp@k9bRSOU?~IyHYl`-eES1)s-}R$+^3Hn7IH5}MWCy4+a)g4$(Hw(egbY-{Ta;SuJzTZb|^46Cx9ahs=sn;e+ zkAi}YUIbSz*D<%HSR}hxO>*5DOFQQ%*n1sFtEYQd2XOM;Y)xKKJWXCOyG2^zT-qJu z4v?JJ25j(+dE)0Ran)=~U8<~Dcy@)F>6^}#PT|8TP;!R1?=Bm zRch4MEy-4b)-^F%u!rFgf7~>YM10JD0IByvqq$#KYAmtl7dbGCBxtCw{I-XfVXe33 zaKSr>(mBH^CHGnrC=!4`u5%5ke&-U=I&`U8_zFp&weJ#b;kB+*A0HP|l!~3_Tmg0A zH&05QU{(aJ?uvtGl=YiXbqc%z{a13}DreP@u8Fld?a|t?z1@C<_k=#dqv|$bXr|g% znz*JgNzCbM^o!aG>rLvFY18)evyWqmt;a7hC9V4C-y~hAk4VRrx+OC+)$=AlwBUXA z^flng4)m=!ZMmLAIIk>cclWb@rob5a(xgy9CeI!dmL|iTxL9$@iL^v3GevRRj*Y2| zHv2G$(lYn&x7Ckr5GF%`&ofOxFI6NjpH?E85}(fz<0GI;K9-6~Kzn@6C*-4JU8%~ejb~fgk|E^m)Ra)b?@LR;8Ws9P{xjBu&v;g zVIJJt;_9Vy5C)Hg!4c3NO~uF0{27vq-wu%LjJAdNlzlLWUuT=LB!Q*7P_K7%@GvPU zZ`^L|56H`sVs|}G+B_-m#P3{F_p9l$tW2?>(Z8E7n%^{OLSEJt(XaZ~N4H0Ju2UfC zT8|9p_f#J0x?GZ7J~;1)wP$%3AV@iFNe*qJ(eE*+Y_;XdC2id|5LUbfqb1)m4U}^GVMm+ zm!Sr8(l)Dj4Qh%tVZF)vtH*0C8XK-SIf_bjWv&DZw*Sa(042k~2q%7MX+My-N}=%B zB_i;%^FtX0P%sq;{qjPNEJxwq^GgX`@^U3MpXw;(s=TS7jkGb0s)(bTiZo27%w8Ce zt*@8xKs%PZgj@)UZ!InNT1&%JikF+tzZO3IyCpX%iBe$-VOrQ9IK)?cN3p4+)FrHV zddhbS^lCXE1Gq{GV*}o^;PpM8a0>5L17^tB5)>Y|$KG8A2=_X{)EkDoZt0%8K}jbY zW|k1u=Gdx0Ukx`x9>~>h$+bEH1FkWn9K$M2gFsjdgbo7`8p3rWWkZI{E#^ElmmqEf z;ZIS~ zyP2=h8MkG8O`xA4I7K>(@Qx8*P~Ss&C-e_;tU`Ax?L)3>ffjl$`^2;oH}=q((%0k< z;>3rX>SJ*U%`LoFVq*#pmn_&KI=Z>H=2O0U8j6-@c%Eedq^%S$l6Q|9b~bExVxFl1 z1^OV9J7bOAejGm+p`Jn|`UJ*2B1$^nhkZSGCD7vz=gg-v>QX@WaBe^^1iIp}BXOvc zEXzGcBGrG$iG|%bzObyvE&{)xyeGD3fR7r|Y7kMIqBy8HFB!Er*4B7JZ%!Hy9qL$? z0fe+js*>EI9xc=Tw~{{;&u?9qT?Y#gvv72=t0L`UeBt1Agx6@UAXUQ)2qG6XbS{f1 z9dWjpjYKwz;%sJOT!bgS{*s1Co!Fgjh;B$d1@T@hOdy<&VVvLxZSwJ5$7C&FRstUS zPSv;)b>FTxGjdUhEc!I(_ z;4gc^uFBptKX^a&uxWfLvy$(>dO;g?Xg^`HQr*(^(l^q-BFT=cjUTQ!U8ABk{;^i? zJRzi5hE*GD{3Db;n0VcH!#Hw_h}qq${7a=ZbgT#=Zk$EARohN_H38oNKlUMZ*8LdT zE56ZmT~d7pV<56i1&v^NqF>(+y*QUlU+&+iFePo&C%a-GP06#fPnub0{qc^URs z7tKfZ7b_rN-X>BS$o@4s-OGUX;LM9_e(G@ddR$S-%};+%RG&B}S&tqH(;;ehE>t}VqM(@UEqV^(? z=}X)-oIAHuGH=yC!L^p5J~muQq3iJSjb9bq3H9H(|Lzj%Hgl>l-1VbV0X2hwJA7sx zi1y40GxXz_Dsz8wsW;jNV^7q3D!OjMhIr!f+g(ft>Mb(w&zN<1UPFlX+<1dha*FBh z^G}>7s6pljXrG{pDIi86-nh>}gYoL_g)YKvPVomIixnUXX2eq<7rtM)mPhoMNb~;P z;fKbv_&~QvKrl-C@I1tZX@!r#xIu^npd8e@4x2D>D_hV!zy8#5@JQiW=IpenfnTwL zT%~}TzN5EO76I+BFo&>M34)D`VETD{9vz5D5m+Z=QHKR-{K;AAH-7?M5}`k;7h%1%-X zDzp3Bq;XGF;9hY2qJ)~%(j}Cuf*C6OKy;B7%mXQX|qy~ zHGwFn#xgceyo~uwt7@+Ie?{hKSdvF_6C9BU*Sh`U(Mc56S5B0{nHo2akAmE0Yl*JH zgkgIk-j{^_h_{pCCsy#k)aZQQqf6OO+KT;+L01IwxW0nM6{C<$)fkBU`~yP0RE6HK zC^LR)O%<4&&9C5y70Vcd$Hj zzDTP?b#x)~%9-`>O$o=+dYL?GwWQxche1*o{yLH*I-=@@9I9gddM6ZOD$uMU8Qr*4 zZ@DQ)B#bGB9F;i?nn_bLwRuu&L!TA3#Rg4{Xm&51b-h-j&2Lxznt&EC+T~8)#DkDl z3WN@@S6lh=V+}5R^=3s5PvWd03-=kkHdr9pcyglv6F$cP?vgO4a+zoNbT|Ihb}X45 z$;K=0mD$ijuTk_mx7BMl2=PfmOgCl5rJg55b|C$TXX+NNktkEsp^}DcLEFtH`BCnsFe*tir7ap{Dj9J=NHO+vd~Gqc;c@3U3Cd~^Yo z(~2!|4MN1uxSqSc_7P#4T!jc zvmfn$Oau>!MsqyLBU z?UCD=fp^I5tXVKwn??_W8&=uOZ3#WIh;frE4EU^B3?jF>NX`_G%2XGl^m0vM`$Iq&)~$M?q%0W4f5hWT@{KGeULj&tI;|n`K4AF83UB zI*Q^LO+3uZ4CEZi<~`u`K9F#X zM4nwL~Z8R+AaAdldXa-a1F`8FMZRXkX&iVa*3jKGr!VW zqJ~HQskOn`n@C~XHJ5f4?v@`kG-K;dn;PcSfLe3`u~&mOLx^J*?CrS0$w26!Z4h}u zl*L1CTF#=#-p`<%ZW-5*$#pO>BIjOw>>}BCclOmi*qxXhqjOVSqAhp|Kl~m@WGqzx za3MfqQcs?Bt1YK02kTgRKMQFn2SckxW1&?!frd%ff^p~(L4!~NH}zEWub!m>=&X&i z2)wmaeO)2(1khkpD+i5M;=9{LEP#Vvs;21l-xdb_+P4L<@i=CH&KB!Ke;yoOsew)G z6OK~ZF0$K#)1vdDRk4j1BfW)~NJX5{j0QfnL)&%x^r>32qJbqZokPAy77Q=RpTm9WvU**eQo=9c2^Qc(9|)jz_PQQMN1wBFr-Xj`@pb1tBg89&*EbX&^>08270 zd##zb&jwvF@Iych@NhwqqvhlB!E(0A=(xPsLfpJd^nETT)LC5qQW7d1tdMFDh!Wl^ z+Ju!tuEvk6xk48bY1u(Rif1vq_nE{ zo$9&!Wle)NO4*Oo&Ky>UphFKd>`nd-!l&tQmWuTzSZ1egjYs9V5xU*EkYC#{_PryqY$K^!tv&kS zr57!rM;O=LtoOV|p!a+9c596bJXFBwYtn{wkX|B^KGv!d1N|8lapHOmK4sOH?un}1 zSxnHjT=JM!%GclL8lj1^x`C`ux&6IV+I3rH;x|a_YsmcN6~z_qWTkKN{v5sug5>I= zc+`TZ>}kKm!j^>DwuH+RYi3o9NqnddJ}ms6CBZ!wr7WY)h3=-?6>CS1kTAhz^Vv%F zVr`9;{H%m7{j_z)y=#*p%&sCtV;=ms?NJBEa^88t(T1U_6;dPPw6v|J77N5$cjC60 z@v$7bQ%ERNFy`rlU5)w?s)9u7dI>eHp%RLhy{RkhlcL5prO^B-ys#hX8!Ab<-LS0R zT?mNS+)<2|#N<}t^reWb5Xqt~H)-LMbE7JD=@vV-((9b2iNB&a%bk*RQK>6hp}|aLRpCfD zJ=Jt9M1U?U+-IfR+!u#u9$GPubEUJ6i(4<2p5$;>U~$oc2Ad!~^O4<}%I@KwzmJc` z&JK+400etE;J)p$(DKTyQ+I^&OjvSZoT*M{;~SeeinHeFd3w~af?Xn`gHz>1R(vH{emg;3UQ8CVH-7pih}({Xq?}+ zCDwmL($yvTcu-gnPNTUx=xs9X2t&_ZfV_CX$MWWn{En_*w;NLE6vl>r5okg-(#fWJj2M$W7?=1OxPBIH?a)N z4S-!I(TXB~ktdZ6WN_%xY9sMWQ9>b}G9^gbSsuvP35Fsb-yi*en?||7uheq0z z5A+(#3u^=p1r7~-I9D{N?0X85H93vB4ym(Bbh^uSy;L5FSTMyvMjw_)9|kMCb#E37 zCR^=q=K!`$9E1aO3#J5Bb$@FnaOYjYc7y3EckYh`mO=#39DC&$nwza+?71El;<;|c|NvXCb_l=Cz z&TXi{Zk(=RC8x0hwoT|PpNWp!-JD%cC*G&etv4tv^>N9{9^bBp2B+zd^q8`F8n|D2 z>QJVWv^OY%HhzBKugl*^o{$@P(0F0*WJA{U-m?ebfQM@T1jIDVR!gbB(vHx&*)bFs zKnITEZIaj$*+$ZbY;~bFQ5ljX)VN4cd|k`EHJh2LyJ=Sfy2nQxid~zL+oOJ9w*QB* zdybAQ2p2^^Gnv@Q#Gcr;lZiF4ZL`y{F|lpiwr$(C-O0SnIp?nP&bxQr^;ZAWYwxPs z)w{ZTtG@5|Z1%K9K>N=5D(FPlVmnl4yphWP!&7b5rZHlj>*K*pdxJ~#P>z6SPQN!M zu^(NMe=n6I`)f@g0UnA5a_-2b3WCM8UDfj8xsJQQWFmH(gOGKBkZRg*_L8)-!rv+L z{J&9B2{(o=6AtTMW>H*XD}ox%32~h=gXSvdE`LPDgRLlNh9pTuwI&521i4J7;Ee*& z18f|&_9-9HAzOnW%%sZS@%(xGXCYi5Um;AwV{9D5VK*=|j@2=_PWjL!owL1E3edC3 zJn%b-8hn^c!3xch9Jp_wLd5?32nmG}PdPb37i~y6!Pb_fI5A1x9jhEJT=_sMO(1I7 zS=gQCy-Y3bG&e8=qWQ4Oy$HMzUY7M5>3<`WLD-!P3y!C5G$3M1US%SpRw3f35K;#< zamK-BdLF(2H^VNMk%$Hepk7+?F56i5mU5SIBfW$bq9X7;#)eHt9S(o21IOr?o`?0B z4Bj+nFe0r&GS#N*&&?Hni`++vbp0{dIUZ8CjeU+4lI6-e*hBGm+Ji*%U zkm24OvtFpNWm{3Wjmg}Kk4dmayQwetAQhyA4t>$Jk%S$%t7(tT{>je+4kmavk(uV{ z3GdIl#~TL#rd`#H&rUV)=)0{bSaTYFZ006Mh-NJqG-=SPgvpyq6@2_C(rHvji<5Ua zbCOk+bZ8v||A14@-LMH8D3?J@$x0maNI_+$W>Pl^;wD=0ZM6Jw_-jWQE7uaZMQi0L z)Xj($rP)mQUDY(QMIzGarAKHN%s>_4tU+JOSH#?~p#ga@SXv>X9Ve5CEGCz0p z_U@g?<7RQr6oe_+&xEFqm9N%LS5 zh8&T_{SJxMD41(s?{6Vzrv70OvsG)1U(0X*=(wycd#Ru*Orc?RZCOZk#m}&l(Afde z6tTJKK3)vz?FxF~HFTycsns zHPYz%n5`0mQQ65P2U&1S!<-2xqUhwl3yfua{swmkGpE)pxbM;$=4j|8O=+-+e_OJ8 z@}9>49+7D8I76mZ#w5lvP&3eE4_Qh~(dAWx0T0FqJILFq`|%-tQY-mkH`cNngukW4 zGg)go8oE7KE`LB;QPXen{?xd88Jdwo1p?LocIQ;gKCVXSidbgpic9lXXErEvH^Hq{ z@I9e+;g_e7l_RDQT+Sm2Ee174w|&vjfVHCCU>C$VlWC;8M8Pzpt$}x<9;+XtiMMy6 zHipmqaSxFvZ_1=eBkCSILFT1D9?{rt7x&#JD_b;8dHR|V(v8b!2-XRB9^j)nYcYps z)7H1shy`eM3WoP*E1NO6p;T$hRkP;GTfB&-Q2}ouuEm}~h=d($humHCX z9ZSfqUy9k>1|#0Y`@AV2uS#hcQny-plPwzPXd9mt3y{<^mdK)BL3Z9`v%~hhz*3ri0%y`)X!N{oj zpd|?Mb?KdAH5?@~RD&@j(pnxd_KHo)=PGoSU?Y)mJx&jWG!Njx>A31ZO`)uK=zo}! zbBExG=yKeu=(t_Y!p<1Z7mjJj-z7 zA@l$0g$s=}Voa0%B`>X_sSx}1NK^*GM0ytIPx&-S1|Q`JBv-rxULM(J4(oGK;<+=Y zaQRdtCaj2T>r+C6_@7yfR{4V#O#7@^UNv4xnI~lu4~axh-~}$bB?xB+6Z!>7wYvH+ z=rCuV(N@6o$D};FgZpMeqq1>Nt{1D76(+diU1jG4kBOK{6jG#bD!=t=piU^~FI!^! z3rywK--#dEa#}C20;N&(>#bkb4| z`l(RzaN6&t8vCIPms8DX`^d2v{7U*1L=9Z4R2HDj<2Q&@Ac=a`!op1Y-HeEix1*R7 zV?}|j=x1+ES)<2G&vf3%Sy^mmsClWpsNfDohJH6$OlSsoq#q!aiaQ{fSv3b>zbMmz zEAMs4zKtKd>Ani(Sk%$;A4v5Q1G`g?lEu(*`Ec>_PCE9d`lNoXsn0MP)vY>vE%z1R zMwBz8g7uS0q2yFKpwD-6Z7A*z^aOU9Cp~L!a~20?Sw+fndL+~QiB$h-O8XV%jf)GS zDVV@lP;yy3wEqoX{19)gjNbip>w#fivK@4Aoa_w}PA9dA%-F%-M8FoHjz!lgWER z%i-y=PCGlGRE?cYn#X78cEe(C^xImo2xz7{ra4Pbvh3mRK@RfEcps{19;895rkWhK zIlMbRMF*i`&jw`{9{#-hc?aV-PE)~P%aFy)T({DvwQ!$MJ=f8Hd1IE%A(eHe9~2~< zkb`t@sxqh9E2)AhBsq*>fxe0qD>MPC)FOqr&rZFpETD=W{Y~{V9T{n+GJ}~zMe0KY{3tmikqI5~15{-n&=AR_D z4y;TbImoGBR|Zw8SCG)LPaB$q*#6QPG9uvOUI zR?6>0kmrnAVHkg0MF^8>?&#moF|KcbIK8b-|9)o_R-b?a}8 z`I$~Kmvu&c$7PcUFUCa%3NHrlJ)C-L-QRXk0(=+M_JuPv?!L*~`J(PlJB*AN7hb$L zuGUww2S5UOLR)68Tn)KQS;7LPXa%J@e`2J*VL{1YNFmHz*dT3d#Vf67-rVN}FRk%m zI8h>W;t9G5k&v3LMIqk>b#O>Erb@2_N7P#8fPui$^dY0!I=;=d+8@%n^qA1RVU~TD z8FIm>9kM85$F3k3RuSw{^UTyPu?EVx$z5xQT~m|B9KXFrPv_gfy126}d`;L?s$<-w zj8b4wh!hm1+MRVTc<1?7Sd*IG2=gNyUX*vwTSUYv{=_W+)y^)qevS^Cz*cLZ^46I2YNAgOx+V5I8Z0c+?Mt|-ofoTh9s?o31^5y!LEe*Y+ z6~I*{pv9x5teF8eoy(G9AIr$eO9zVI!aGf9{Oe)+b^K`y42V=isN|hCKp{_M2rI}y z^xQiH-ICW~>fT>*$cwMe!+cH#&Zm^KY*U@^a`u|IqlPLw-^fg*2NX6&J$61^ z#zZ|5PPOK;73LXnMw<@x5w~yKPk)q4Q_VfFha2tXL$}62RD6SxH%q*zb{g?w6q#FJ zBx`9Y>Zov+beFST+urBP{0!%%6y^nv0lDdB!)(CQ`wbjg+&BZvCe!i942%ne&;pCx z?y-^liio9Re>ZFC1FjZ~jgQ2n`{Q@f5R3hDJOum9dzsuqmQsq0EP7K`UR0}Ta>6(_ z&SW2clO619rN`&Fy#b1p1p6+m74&ztU3&$SSz2_KcM+S;vAf*96MjTnV60TE;k>tB z1Rrx6@H4laV>cm6eI>V5#3?22gWk-+B;<#|(e!mo+Nk=AiX zddZ*1N;LB-Z$mGEr}dsVlKgOMEF~+o@B?!&X?uZw`yh zWyZ~Jjzjzpm*Y6%1iN}=&uj7-fZ&`AuN3lGZZ;O1T)LjGB~LDQ7>2r2ZE?Wpi|ydB zUx>#V+B9P$J$0CuZ7$fnnq&wDfd+qh!!}!9BL^$hj59L9wR7KsKvxHIchZZlvFsrIjweO@p5|4HiIU?tf!8- zlMvH%-El^L2bSyUmQlNWrz8Jsj+Iy=G2Hd z9giuBLwDlsn4I<<+qeO?e(W@{p?PLhG&S)f5k3qXz~=Ob!G3;pwPY7=((LWhcPok( z5owfh^B0#R$r!+CEXKOg+~P|vk*C2h5DclV-$kW|f%}C>8!=!_3O`CjH+_fkG$XQI zFO%=snZOQt0L?j}jY?}BP!#x`s4uN$u6E0z7p2aws$h5)wZgzbHjn#Vh-gL=4aQXv z_lP`%_)_F%0KMicF$1NZ;pW;YR;qng=}GoRFMN=7E)YM;%?puKJ14Wrnql9${m!?= zhB*bD)9oWt4*k6OX&^>Swew}_fS%9pZ|cL-W=A1O0gwUMpRE$(ntxqp95Ak{BG3I8 z^$x5uUdcLNeVJLi>WnDkhxEyimz%+9@d+}=$*OT7 z{ZsEo9J@`>7VfmymG;h%t5!+D$C_G(_ea2zUw~s(sx&@I5bV2&C<_gC0SBh;|B}UuIwNEQ~!4=wPOHL2d~?r|fWj3Ah6WsLBhU5J96> z+VV8t_O(-&+c4OW)c(~Z1r!466n)K5WtUgY(Tox2DhWHe*TK6~#+FTyH9?C&&g@T%NOQTK69$J;Q{N%BD9;uS|L zg$qArD{2up|NWudg%5;OIGiq}Wm9nGr_;8srkmRpCbK}GzcreQVT=+joSNMzU{gWO z^uUsEtI+aGvDDpXi>DAM6r=#fYueW0>p z-2}y9{rAW9SXEjnlqP|T7(BIIhGN>d~mJ+s@7;r8mh}vdS zC9TNBIwX*}+mbVcJp)crnwquc*{i8($4l2ShGgA)-?leRbyt31uG{SHrAdi8rOgBd zC*ET+l^eYURRGlQQ`Um()oWWXp)~+c8y%S*`ImORi(bL&4!vSKBq1XbP7w+f9%G$y zTvK#pBU;B+jz-(OFzSW4c3DnC>+3}gX&16M*xEn2W!&Oc%qCfi6PE<8vvPtpwZ-*a zH;y)(a@`BKcQ+3XZA9;ICj)L8-qUUX3pkswfj4u6shiIdn(b{ZR_+v=K5e~rk!$$47>1t76V%b_Fu44Ozm8vH3VO~E3EJelLd=z)&c zSNl2pfHP^!83s|KxT~^iUX@)MyKt{~PePJ<|9gK{Q(wa8!F&O+%Q2qg>dk&LEi8%fFvWC@AW!QZu7;C-kF!2E9fnTp6x#1F<6&@Dk1?V zz7mH1R>Wie%+d@l|A5P3jw@g0p7zmvt$8(n^SqU_LCHPLzS`36RPBOe57W5aQtOm% z%k;5oUAMp^(QDL;>OG(S&=JBVGIr0bXN@A`Wnz@@pnhklzSy8Tb*2p+Qe!0qQiFlq z6n`33gkUl0jQU>7n_OZv&$W@Tdkf#Z!zG{88(0|81y9p_E1srQWh5J%0W(B4Hd?8- zkiKAzkH`Dey<9?miBn%3@-ScytnxgWq{PWH%x9-^f664#eDtU{(P?4>uv$HI@)F>K z?5;3|t3KB+OYA3o!*^kB9~W>mYiy&vki23X6^YEF2Ag)xD#l?k7mWmcDDV z&7b`xslBV*tKl}{+I!(6B8jtzUJ0xvWed$O-v7A_Bn34wZX_F%cdLBswL+=s>u6+B zcO)GKej#u@?#g^1{OIsCahT3<7Rq)3zU53vH7~yRmGl*X(nta4|LLKP)ejc9Np2VDhzPkZ&1g2YNEVs*W z=g`U{!+zm-4BN*&=s0aGw~xNXt_+;!9@TZHzu zC(h0BwFQwO0v!9wdOdfj`=Wxqah*qYT6UbC+4YasuZ#LTp3S|ejRpb^F=2RN^X14)3<1Jqx=+v28$9UiHkCaM`TG z7E_It*ijvEfM!pcsol1TEJoC!@`|Ehz3WW+OLRCZ_zz`hDg@OIxiV6Nu(rf9weL-w zL@wz^S-MiEM{n)D-%1iK&Cy9!%jy!xl}+X(`uL)Q*|oZLmfiZ`Sz@=Ogm<)iG9}l> z)6!Dch@R9JnGNYtKRz=^O9!d&5^cF%be1KtZNBvKdX^>8gRK5`@zfF9b*HgDT~wBz z7kAvOv^A#CgT|tCcPFdX;use*`MX+c%ASHIa0WFG8+Dv6>>d&`G~EuDCJ-dyrBtrg zTK1?3Ji|3K`sTA<0@0oX4m;{vTc<2bo1&Xd`&_xvSrD$+q2hm_QHXG;EHTvz1Cm&QLm?pL*URc z)yn=h%3HLs|nls=Q#ciy^ ztWZ6Y^cdy$-9Cr{z~R`@!BZ+Y#?od~WvzunY}`kd{$kg%AcBgPrbB@VrHDERhwfZq zACUNbe_S*z=fK2{fyI*vQR|l}pR=^175uL1*$vpNH#1cU}+uv3rxXh<$Y4W)=8DiD8>aHV?s{(;(2ZW6Es-IQz*_{l|j3Pt=4 z4|O~Q#GyC$%$><2=UOX3v%9!)y5$#iq?y^pb-;>;Et7~0E1dkS6_JO?}*J2XA3pD;Rf-NSJn@=h;ta?&}8Z z6;_PVh9LARJGpS$b|6KF5>*`x>}g%ZP~?MqfD6dW@8xNA96kLNRDLv(aWPoxuS`8< zm%B31MWiIt^&Jks^NU_S-%xJESda{LFBTkj=wwWJZkmg5EYtjZy>B$S!eYN?=cX6l zH85$X9iW5@7Eui9DgGJU95@M%;%&cu+kNp>%xATT1E|xaz`izVlc$@tHhtmbI*?ta z0#}A6yX)loD@yk_8j4fEIzXtxlw`Ksq%0?kG|VZ&?W%=-RCm@i8gYS^KSBp{n}qry~!gy%>l!w>iR z)l)&hflDcCNop!jv7wZS_lPWTQ^f<9_ecBI(KX%?ex#&ER)0ak8L1A}q-{i~=T|zs zWQ#3z)p1KV4 zHZN2s^DqKW8g9DRKgJN=SgHoS9Z%7Edk(U=T$7zfQ{6mH(QG*1Y~KdKHr}ISVHL>{ zTe&%Dl9#$}x>43)kqUJ%L6IeqvIrsp#&_MEm=kMy1{Kkq@Kj|CG zdYyesR1qXn6^X|#6u+F8&y{?Jw$+!o@LcLK_>F-lf8B#tvDi1;>!IQy-8OT!!y(V| zI-OCTB#q*GNhmlX@btL7o{OS;2FHBIkb~@;Dk8i5=s|qRdD1z2!h2bZI_bKF`#6@< zdxuHlCo4K5}BL6Ubv4}5Nhna$3 zzrfECelK=xb8kCh`m4RZF}smz2(rI=;@Nuo+jd*KXSyrr(WRslr?hb3u)Ov}hs=I$ zte*4sRC}%%n)VdBAM2{FphHOir+rp!z0^MRrL5;dABH4dx=sN zCb=yhDWaX!{J6JNwD{{ib&spF6On5bXW3LN*wkw4TcwDr4Cmcp_{8huqV~AY@zaJ~ zlTM=TApGmm1-D1D{aj|nE$*Z0ROGfvtEqFprY@R&w@6l2GKRceh`UNaSr!Iz9w2bf zTqz96ei|IB)lf%-R~b<|%AUoM3Ane<+vS_O;5%@=6S11UzP0PZ@WXbwKDCx z6Z=Hgfy%#2x5?KM=2qnzm&Ek}^;))L!$-1Y&vdZc(<&}ymIi9x|MQwl_`9N@PBVwS zNmYs3bHSpnXl3nO^HBRx^R$b6+gx7L4@22TwF$Cmib%I#X?$OJm3fvZa^y@Rb z7bl7mgDEXtIQ;&R=W#9&!?_kF=H0%u27N$+F&* zE#KD{RT1ULRM?#^*A{s>LB+GPzuUwzFRW6U$$@>Hj9ydjTkfFVi+80dS_WA;-^}-Q z+RvML>E80&%9T7-iJUa@Tw&B`9Z++^0=ly$AH*8)Fz zv)ABqIJ$)5T?AvXIq}J$GV`{~_5p4Iy`(36yOs}9AbVLpxVT?|VVTHg#FuVc9bS>o zc^_q@uP-$(of3v%fx;IG(o9%b(m~RXc^8^{nFN z!Tz)b@lZtz>--`EbSLOC&G^ z5#LTeK;TslQS;hep=gkdJPJQ89hdvSh24<~3~7j9?1M%4Z^y(#M#HD$F-TcD%1Ab(AbKq2{c3M$D9lYS>vr#R}kYg0*v}5 zYq=UDt(F&K;Mcfn^)Mi-_Xa;+$JWt z!+K&7?is~n8vQvYfylyk{ZaXq$?uxXE|IdCMn#I)a+?y2q2uhgjR8EoB+7IrUUq|8 zb&D=v7MOLF>JoAVPF#8IGLpGQ~FwGG~ zecnh6Sau=;8^of~6Iur-jB0`ztt@xmei-Cr#TT*M>eY6J|FjvEhie)A$@&_!7`0b4 z;f+vYE}oN4je4Vb%@;W3+Rxitj8krdZNE|vdjQ>U%^1?ZJ(4)S@<7vVo~zkEv#sw!5InpNg&}trCtR{>>9~j=)%ln{DbL0aq`Tv#xkr!K`+i0 z+~xfHcn4{I-Y?O(b`i1z-8ih4C&a%2=f^`6= zuZdKuu4G7FBacIJFsy@Oy71!eSw~F52os(w$cj>9xwG5;>l?-DZxl~6-=;?_FTemx zg}gU=_@i7yo1c~8Jxpif>*#^q65-}n0N3!gOZCur7RW0RfeJiX4bIcxLmJ7)5O`x z%0B9(i4DtqU^wM!9xVt_Vfft=<5lq*1n!RA-?aCeUo{_#`t`S_GpCj2ttSKy-n?)E zsNsTK`Y^3-90*5@o_FkCV%5ZKF&`Grn8mA>{$Q252q|8ur}>hL!LmHe7HUZD?=@=R zW%y_1y=7k+^Fql$6e}^dDOrMk@;q$yme5Tv?M$#RDwR)=_IQ4D;)0BnZt$(Il>^V=!co_&R z>~!Je!U(sHgUaHj(;cZ-hAo_T=@9{aS@2KMq;@kZW5Wu=GRGn{QF_u$F!x;B^ikdy z98GcrH;nflSL|}X&1fShJ!a_6)u)X%6L0ImdFy$DZA|P{Qbn$vJXV>_A-0l>eZ88K z4&g17S?Y4_d-(@}kc)EPDiE4AkBN#ax$G`mP%8Q($ydWz|F-)sRtKXO?5J1$(W&K&X^TNt#P)lusI=6DDnR}_c|AX zOFLrGL21^+fpVhty{XdK8t^m6nlV_cyu5K>7(0cLxRWfSHk?Ex7Hg@Ni^b2uQ9KMs z-H~IVuwpXVO(;x|b|WBZ#A$F^)IoyR*o2mrE!Xru863Vg5o^q1@lWRL_C|9e{;YI1 zQ!3-G==iY7TuOOi(QYcjvtteNgi-N$yvcz=P$GK;vYCy5IhU-aB1aAwyjc|(b08D7 zZ9dJM&nn{=hm;#`!b!|Lnv!E^VXUi|_1k(eR! z(c&67Y(sG6J{pS+-~R|pVH38+WBH%H9v6Q#kRE^|GzeSE@0|@V%VCxvW~>z4s?Fa2 zJ`lr#N}&_BB9k~1CKx6tJZ2oxOe)t-Qhf3qSr01XK1~e&&j> zJ4i4Q(Gv@Tn5BDjx-k26(dzY$kS4LxN`ZqU|2}!M}*Z0x}+e@5d?X5PG2LanH zlM368$CyplGq$IyIThRGTAPhRKpLb+tQkAo$3lfUR%zSF$OEbce>$=&31RhB{i&y9RAXEhFC>p^4eglYyGy?Eeh5A1d z0mYn}dNAQ#6rl!&7CEcQnZqakdI@gm<)!U_wno-KXFi={@cR%-$e@5qW3h;2PNPAS zM=dbQ!f7FMOud#1pHEAA^mxWAm>m<1f@#e6WgpWkSXfSXXbbFA# z;%#UMQd?c~+$RwzR?*@g6vb_X^#?5@;j^B!n4vDL9}B#0AP zTXXY>v!k@*R;aiU^O?gg^%zqn~Z+&KFY-{Y|fHUszo44ekmP(icF5OZHh z0)6T7ebN!h?l!JL5JBGppWZQPW~2W=(Ap1S;6ma}abyObNen4Q4vpHKU}FBr^bTu| zFs~0cUq;GujC8DV%yaa=^Jo^!Zj7WfNDm|?UjKT@2ib!ZMb~S@z>~)T^`hGb0ATzd zXbINO^!xvn$o~J9xt{qm$Nv9-JYv^u{2Abau3SU4oH5WOV0gu$N~TqI4Im>fzM2_u zL%UE<7Apu6MigOv) zWv_#xH5f}x_I{mA7f8lfhBrl1k#TJ`13HMNE+M7%?iVDFlsd4U8#a=;a)Yj2+BvObIFeSGCWDiU2!1D`V?_Gy}uu*9C@Nl~D7)K#PCI z1~UUA9TOWTt2QAs+b13I32iVle4-1V9>yo*z`@A&pR&h4=7WWkjggRzoso{2gNXu$!c(XmSJY1V`pUHAY|oWq+?}a{;bIOxq#!}HdxvJ zQ+D{LWPG9#te=DfE9YnNPf&u8iSrYC_$P$?>*c3eL&wR)$oa1hC&TA*CPG#=*3WJK zM=Sr!AYox*r(?I2`1Kp<|!AQu=$wtTWk0<$8>z_FB*#P@zakkH1|7+{>Nsi!OTI&$@Y(>_-``fKmCsXC1x-(eeTxJo$>!B zGZ_AFRt6&j105>|E9WP^!Ny7VS(EwGb7A77)s(~*uQ`ENfro4?EaLG&=z=2hNP>p-XxP8C?7v3P z68=PzNBT9LY}VML=ZwlahFND3E}>y4IxhWKY=4`YER7-)zGgH-)2U7t#yI<)QQEo9 zl=|e^dGvP2J9Yoyl%WYIJ)K`EQx^DA-3^1JFW@O`{9fcZX!GsQE6$fo?caP|w?m!G zDqrB%5%hTZy4nkLc*^SM)Ziu@78aG`*MKXJ-}*fe2shMtz2(Q|F&;aF_g21;6<1c$ zAbM})BVWjU=RE;Tu4SFT6+P(NDPP34lnq8sd^y&E-^OHa>a_SUMC4J1Md~CF_-Hk_ z`Kbz+>ZtraDYoqCUx*2TL~hbcJWwgtkIc-PFbAJ2(AfsQHzr&S2bm@#A30 z@(QjHh|7|);Q!0?;2tS8Z}SJH5Kkn0A*5K!xHj>i*mX7jYvy;VAAWG(j-w$I%`G}X ztRpa_gefy(<3C%m{jPRUcesb+Kn$_lg09g0V{$^gUf*e19x=gcpx0R*Id<4l$H`q( z!XYzWVMzId6x%|#W&Lpd$f0b=nlrLP^pNvR9ihDwwp~*VQEYvv3J~{4oTCfa^&;-U zrQ_Wd+_`(^szvDvp3v2Z;Q}EGaI2-Z6uoI5{(eJfNnV9#kY5$OK)2w}VBCN+)SXC!2L$f`arfQkUWZ$=pK9p%TE?KZ=_Zvq&<|^L3LL!&x=Z`+oewdytPWjnyEm*j@l2>GB)Z2~V0y~|d%@`*8^f{WM5ud#|oqYsd{ zGqa`d`HgMqD^b5?n}gn1=Xh5)*RqnrJ=OnW%A*Z~8TF(K)vt?xRP(brCc8GQDa`i< zfd5C+=RSxXnSLnWDVdt@oBBLHsTxg!P((W(3@1q+N^V=)F(yx$4(zuRW6Yg#6Z|W` zHa`L>V+Q|gECmR-1VcjQRk_C)e2_-DzdN+WKGw7xr=SC=SMo>7Mc^T%H`<=xNdP}B zFAgYU6~;vImNB`{s$Nk=nlBZmZ(e=7u!Y#nGwj& z)iY#20;o;{XW8n&>~uZUhl_$ofUF{{ypfz|$_CS#KHKPdz@S_lfzuwBA&5Q*-Nw|3 z?uLj%x;omaL8Aq$5M`VMvuA(OXw3NNz_c^#jdF#yAuERe!Qi9cDs{5D)>6*Io$;t) z{9}aPY!c)9TQlzPWvj-m+zXU#oBay|&bigf%Lu<3^{A*jua%+mgjH>sajl^#ah9={ zVLjmJW`Eo3SH-@+?>fP-4)+=zCtw8DAw;#-a9!XT-T|ebsbiEKe2Eg!12C|)ctbHl z8i%Sx4fSiOML>z-WR7qVwv=DjB<_Mk4J5io3S+H>z>Z>_gKFy4syqw%CB-PV`d!uG zL)IsMJfLQ_ZGQ9n5{$6lQZ?TRD@C+ZvK(lW@<$KYEJUb(dDK9^CXu=Z9yW$;Y+qbezx%1zc%;Q)2wTWD|a!sc0zkfM0RHm?Ftj=k((6oEFul^E>W4gMAK1 z350kwrL2Qe>P>^%GhcOxvpM5OP~wSLyhD)NLd_<1!__hcnQ~bZekKcDwYEo_IW?;k zsBB2TnFtm3n(>o8<1;O2zN8u!@Z2s z&@iX^G0{Z#h>^z%ss$E#J2%9?Ew}6k596EBhw{@acz8#x(#~JzJn~w|Mf0=QHRnke z@VEvYL36*n5Ao>+8a^8;74hqw2V*qRY~7{cz+}2CR?Pe`_IT4zf<`LJ4@EU>=0rZ)4eie|Zr|w#0@li)R#I zL~AUipZ!R%OF)L5gQ_HHmVHIKMCS8L!$j&#Sam}`Q;#uHl0i^GlsBG_5b$*rNgOt( z15IaBjD(h^jH6p7(`q$|{o=~a&0wtA*sVO?B;HJjKE6zur{*jGW_6alzZSt`CTB8I zINGT^M(3y^v{^huO%2O)wrsY8{ALVch9YtJ^>_1gM-&TXsp(Sq_D)ke)eDRd7}^YE z57tXhQ!Nh{CUt+Sc2(!t%g^l|CXz~3X*+K`jH#~M?a8oj-4-n-G&NSVUOaMNwx3

I;< zr0e?7u(ajkxOB;Dd-49Z_P60G1NhVh`S*FdFA4z{^c*Lgh0o`3zic0>Id+^c)guhp z``wJ6w8xA2;}Oa|uy6OlAk8K1`OkBtQ39DbbAiIiC4STWbCMKL_)Ll&QArE!AY>%uBxE2|74QLFhZ6XUl#z$=CnHsjra@GHX9#$RaY%wh z6bWy}{Zl?Lt`ns5tn;o71)&~@fMa&W+-tM?MtI@LeBl}VoE#7*^K8ElwcM_e2rHgJaJ9`9CIla)yrIs@z`C*fJ3!SmkeerOxh0a z9Sq#*(j#aG)9ulEhCB~``iy<@+uF8yhT`)<%)##TwcSx^hOp4X9}m`0`+}2=fG)(# zB0$X2^Yil@=bBZ*gp$VBa)(O|;^G>LCiv_G-4QH3Nb&k7et*=Ap9-8XPAJ6=xeFBk zKp#b(-%fEJo-&W0*bOHBkoT@Brf&5sXuui&jc z#Ao6USnu8}30BB!3=o>!o~;Xn_fQ>XgzS0umq9%m=RS^Sa*rTYJ>V^RH$EnPHX&3l zA#`mv6jdw3R|*CFpRM3B4El1?y*kM>$>6X*VXiqnd?}Z4-t=^Pr~;tbh_t4`+mMKU z8Bjg;eI%1{1W*uZ*ugLg(SXn(w*{n!;GR)rp*FS&af5}gDRF}guNiR%b~wKAa-fLm zBUtrn-5cN^8h(u&WK{cOXpb8?1kHg=u8%A`NXda#)YGRHPRfC2y1l@G>d65sRRcF> z&!)1CTi)NI7PwH2Zh4KJ4vA<$J2?cd7W~kRgj0=-p^r*8XkyhvS{+e(vnfRI414aw z9*A6xr{kk-N8FB3F--*L^S0Bq1Ml)}Ggx(pm8Iv(4!t{$v%qfr*bl^GYcp9KlZo6{ zh3$MWO|+u!&ph|~Bqei_Fh7@pNw8Oawxf?;C!vU@jshN_RK;C2esHCHnB>+13-ek7 z1abo6vzFu!f%2-Gfh=CCSmch{;wMC-Eg14{Lh+e&lJnYqdTL!tNlO>ES7U&-Rr|qw zlwR8@fE@5e&fWues6Yoq8m@zaViq)pqnL4aYnU>2Hl?YxaY#E>@X)9+&Uu+G>DZ;T zY|)5hK{TGVTe%fV*%w1B6bd|6`C8A=SfML@n>ckNm#(xs5g{d>Jo!c#Ty>!80HV?z zQ|@yKQm(8hscPIw4P)E4!X%rwosW6F!uH zl}lSoGGS=w0BavFP6=S!suzU8=BK?5>pmKRPb%>j3v)DbnUoCss^8az7~#YN8ag~l z`2s&#hW-OBPEzQKE*Y;0p2ys?=q-?m?bhN*d|rvIHrFxV(k+GCw`5e)1f;Hox2Mav zwm_edKlPVnh5#G*GN5BzM_sC}sk3r8IRQUnEg7;EC-tXjoudqR=EIUpPoF0|&%h{6 zbyL)G_KxsWjPw=IoQz;WmUK-y$kIhTglLR5NaLN#o^*3?+n8_=glS~iLY{CQ5WZbg z;jd*sW!g=cJo$Vrb`-7P=DH?%IQ}q{G%!K*mVQPRB6aOv824P*CPh`mOxptRzA~)W z{@!lJizy-!`;aQ<_o9So8|DwV zHa$Ey=wU!yo;m(sEFqA3g;f{^Xn^RT)i*vUu7&fsZXTmP8hVzipG6n4Wkb z=+%_iFj-XEShBvw1=cW5Oeb@}as~I3+-jYonqh4Y)MM`5_We=~X`AKUkYU?TQR|%? zG@5wOFc*bhYTSvlPHnSuz&Qh$2Df%Eu&p8*$>KYtpZ^ie+2jf3v9@-#QPt5@ z*K~Ea8?gF}cWAPvrLih|O02P#5)ZiO!B8D|x%+>Zd#mU;f?&bYvPBlNEM}I)%*@Qp z%*%b!Y=D#d7{#h zQhS@%OD@aXX{?^Z4o+w~86_ynW`O$X28~V3))hD|sm8DS90W-E;H$qIRVi;eG{*bOYZibA2FEX@)J} z%K^=m5uBrBGZ`|M+Tu4nZBFeiLVN?7Bgv3c5~qtWWEv0)%3kz6q68WRT7m<=lOsiw zjFMcT1lJ87FnW|-8s_DX>?rmf$hx!fZrth2)5u=3;n^zb!ngkhla5l*ztq(4eXqU8 zS?cZ7bF9Iy_ZqUAQ;u`jjk{m2X}242eg7M)OR@ZOve-HJ;LguBQ1`w$nj8Jyh%DW8-3ed!Y@Qsn%QiNAQFI#EW*JBnMzN^g!Dvh^ z=GoaSbj_mM?{TS`);@*??RC8L$PVwz{Y=@wm++dn3j5zXO_U2dzrBHlJ=Ja{Ub3a2DYv6ds0D)RoW-=T64>~ zXj-8{vK6Lmk&{-AAr;aVYYJk^1tKxF0Yg`lDnwluJ<}wVouLnMh^ou;jAt{suH^ZF%iXW!)=@z40@eryvwFoRY1 zY3{*TVua@l(aSf#*0&3KX+ThzC}Nf)l5~G9c6G__Z&o5;{RSnnhI%IX{UI2g*rgij zI8|0!07{g*HCG0!kNUD`aq=qE+$kWt4q7P!D}--c!8DplxI;{{vY85zG7DWfF`LT! zpu(wSgLIpCn}VMxYY@#kLUp#9aS6>jN)<(>FwhL^Mukzn(1&_)!phs>!%bHm<~wJ{ zrq#jd`#FX*^z~qdOzSZv}aI9`etYA6{8*dU*T#tOk~_E()?bfYZqlG}{5T@hvu zNk8xyph`L`pk|+Y#)K90$+sI*IDL;c%dvp~-@DH_!OGuio=X8o zgKhS0@SbkaTiKP~T>+-PLGN5GSbS}%)F3!2Pf*A!ExhVf1(~&`R>M+tixo}_?;6;7 zu-Tl+Xo|BcnI^ITqj3N!l|nvs=J?%AbBA+Fbnx`V((IW9|0t8 zuCL}px`3B}G6eU@RQKFp@n#1O02|zm!wg*Urd5`5qEGObou*zsCPrn zyuQY%GH(sL`+i1whH`_r81!}+N4zNqdC9}zJ|Fz;8*4b`0x0skQNLqR%r`|mF(lpD zJi*rspe=YfbyG(^aUF^mE(Z$*vv)}EG-8ByRPLZ9hR_n}uaMv(sUQ`KuAV|OuZm?& zAqN^a(+2azZA3-zk!@Rw0s@1VfS>E2PW%5GOKR;K7RzcZC>#X7iOn$faM39 z;t&|lHulwXcW&z|Y^&BR@kxhLEj9On38(aQnrS@nPGiWVDJr1|`Cw&^Jl{FPquKyD zz%hGnh^Y;0yCQByGc-978>L>Ii7{6$dmS$_(!xQ`3mF8Js^clUGh%VZDt!DQfNICv zRgu4JY)#@!;GD(t`lBabb?U$cZs7z3fa_!i%SyH{L>fh*!G?sjo9JHztMFlU5 z?KBF&2No(Vh?p0~bM&YRl6ACGq~7(RUcNQ*F4E6 zI|KL21G6)Re^6~-!aN9mI`h7NtxwxkT`qo`0Ha8u?^Urpsm->gV8N< zvV<*oT7_!ajSAMKt4j%bnPZ;y1Az=R#0Y$XtXQWWvCx2` zhr4cni#xY3p{h26^P{12J^;vpHt<31ce6Z3Zwtr6?yPPnFC%bf@BvP{|BZ|^PGawf zLf*ToQEjz6T%n3qIk;ku5+j(1oJKP;zKl=?UYtmuIu(3>ZE4{LtLT&(>;g)QmaU?C z<=+L_O0tC^Jrh3v3;Uhj`R=u-6z!Q$SXyERx)V_dd#JN^D9tjUmYFNi_au`?gJE;a$nXn2$_x;Jq z0Xx+6EAAs)816GnSI(Piz3a;}iI4XX-n@QB*tJZ&@Hc&eWBq4%NoHOKSfR(3&^xHu1wh`>`zMa;y~hto&Y#ijwHUJUZav>cg)ZcG|$ z3Y!Z=6)o95qdGE&x|}VI#q%9sH2_T9gB9mO`bVM4+-_Yuvu+SB5wI|uYDTLSHpqca zT})8$(}fm7(?^p|m2-*+OwIcyOx!TUL7Mq#8xaYywE}e}HN7VZYwP%DwXM+voOE@dxp1l~^N&d%lrzlY*yt($GNacE66S7O5Dq)JYtVNr`z$2E?{ zeHOB;Mwy{fwfXP!b>#3e31(3$CQ%GolWdCsEdA*TUD`&{70&s}O=yNWMK{NlM$Un{ zI9Tv;_Df#k4ehzad612QpQpwMBAGbIeSs;jK;Wk06=4zF?Z>I)f(>IUT9!qpTJw4} z5MixhGu9vuE~|WiNwh7~HH54rPNQI|RZl`|*W1`jB2%W5xK5+qVP&CW0Qz^^Uf#Yt zhm-@kZg%%22y}e69=Ht?;o=2fdu`$^YT{3dmNoTw>1P3RP@MwcvP>$LV6ekWTqiW} zzS541(Zwil7L!z^9SttJ1Qr(jKKZ-TL1qGbhnHbmw!bSkx#4DZJ-g19MCnJ+)wLXG z4{uwnd>8%;SDwE%#-~xob^{DmPl_m%Ycf)nb-_?#KPUu~ie$i&lBBDY^Dh@D;d1G37>M6nOe$;sW~e8Yi^T5$8LFDEYnnMIyM!<@u@$vos1wjRO&vMqhLI#% z0A8;lsOVhE0rBENG1Tl+64}3rAgWjmY$7BG{>*fWi2Dmw#YLdGW)w}6G?TW_YM~$p zuIXxzo1gZD*z>Bs2^>O_OJ2bxJ>iG@4Mj7AoG(au2P!#Gwhc0KE!d79nBVtDfB`xH z0R9*^-nlWwuIoka7TndJoT0#a?efg0K3A0cvWghcqNdg4_^Mt-uK_oygN0Kx52y^HnSD@8I zvXjJov`g>JI3MDf!lWcHJ*#mx6}MRLl}NO~TXNA~AU7 z$Td{#oT@%)D-XfiuS6Q4*=PZS2sjcijx{7z=1e-JPRps~pk%LRi&g%lZSwD$&oX$_?cp`vA+bu}q4(+fWpim3GwDW`HeZZ61Ttzv!B zcZr9=$US|13*!|wnqT2f>2?o3<{FTwkar{LWF$giW?&NbIy>$A435G4t)(1az!Wkx z?{$c1W4_^6WI0ao0p9X=_5>4?NZTMuCc~?0bDPxAK$}P=vJ4StjkbC|!@~5hKMA2# z7>%7dm=MvlNP|=E>>4x*LTWV5x~ikT6#zbthL9 z8mo4YLmF59+RGdE9rc?w75Re*`q0l?JANPcRXazK5%m6}*Gp!nK|bA&SvwAw_ag2G zf80=`GW z-d)Ss`JWkP{-!%9?K`@$RSR_mV|`>2!IvDhioNJB*zMrmv(t{Kjty{4aJWQ}dF&2a z;tW{r(BH1w1LiY@j&!h$g5kcY_mQDtw?pyF0*g8p86$TP`8et>j*rQaV%8lduD>z%JhcTl*R%?`@lTYO$I z4~{wO0_I`Hx6D2G>^tj3&aIqlT}2W|Mk3Ex;?Hr8r!xSES;$mS=0Z`Wlyv@&di4DtMj0_vVun3S%hLSstSoLEh1Vsmp-SOsb zRX!dQQ+|Bw6rBw=tpBBRcEjR)avt|E4z!P=lP9W0(?{7x(=WKJiT~!RfG8c_8p=_9 zs6xe3#NP7F5SlmW={NSNHQ}0AU`KZ3ludreSy#4~6}0x3H%k-J1ptuCeoE|JP12UE zCF<$>^T!`7a)6eMo-7Ntmiri)XZr$7kwx#aZ9=+9&Lyx>u%$_pj@X@gEB5gn_*_pt-IQdLrx*f@9pr-2xc=(&7?s83+^$M%&Yi*D^%WY7##(nh zPsFfVchWMEf*^(cVm7oZqZ@AMV%T%rE-Z{ekQe7s?eOINE3l+q9jX~W7p_w1>NdTC z*b>xg4>L;fLxr$;r(RFwt@Ui8KHm#CLX{nb6`ISj?8)4kf5*&dI`kjSHjW4!vuCW) z9q@g=X@2s#6vMa2aNK-VWP6LeV@C8VNa-c6oKThn+NTlF@2K^ri|6nca(@i^_hq2b z!>83II~@}z|MV=bDi*RFC8*{)@K@B_e?!bhZ$#Y@&^m~-SdrkBJc}#lN)^+jr7Vl; z`v4oCWf=Ki0Ez7X9p(Dpv0KcH|0BAko49Ee%zzYfd5hMup-1~JdL9TJbsmWMR?v#L zLk?t9_^>~_0~GN-Tl3fas0ES-0|>PlfZbZg!MkfVJ(NEXtUv{39edsVeBC{2+IjPcZ>HpjxeyxrisPuG&sWLNI z==DNLI_DX^+$!LKz2T4pdX=aUMg*4=j>j`89TrbI6(*OHqDbr^SByr;iNGuQiBL-y zh93ViY*Tsq34F@}d=O?)_4tHDC2c>C6Z+3m_&-ZYT^M>% z4;OJ|7eg1*f5(b55HkMLy3&8}od4Lvf3;$twf_Lge>fr|A=5u>^4~`$!vDr7exjfM z=ucTw6H7xOdyjt*=s#4GiIAO>>GK4_pS^rut25z0F7iJ*Hnsc2O08i23yl2#Khge! z|NM`u{XdfJAG`ZsW&1C#^Z%u6|BtR1|8EWcpDODg{`y}9`ltW@0bTw7#t#2yf&SUy z|4U^t{x=K${44&`692i;|0JCAKht&n^Z-^EdTC3OPk&=?|J}&R$imLb@;_j&DZb~Gl-+h)AI|8P@Fu3Ow!&1TvT}ZHxW^F5 zYzh(uCvU>{qq$hx*p}Uf17;1atv~&vtK-C#9)X4Y{+P#gYKA(8ksJ?$1YL)_rAQ32YUZWg7-K5j~md3)7y33N1WJq^$Pzxg(E~3 zz{g&}br#Bt>&IadZ7|AHT9MwRe>cP2YXJH4NblRh<%d-9L0s=!U2m{AQ4l0d%iDFE z|I37bXKnh&ErI{b_GNEq)w_;5KKE1#xNN(hP?Trcm@1$-kJ|E`+QLY|`QU@t@Gq3k5dP@EIfbrphG=vTgMzmY<8^U zJ^;tIAYbv-C9^8jZ&DR}s@-odC8#T;=p$LDiCz8oo;<9j$KS|+){ceaSk!U(=nAi8 z3b9OSz-R}`l>szv3V^=kX zpP0=QUfG;F*Z3bNv}MkR%3;a9%3(p;5e}Y+c)Hog*J6RcD6gYGW$s$y79v3@r6TGr zF}GYFLB%KhtdTR)K3ow!4}zf8oT^zJ;A=AaOvs@WiPuQ3)$56R&BeP4SCpGyAQ?=i znx7H}{~uz$g9ZA+@3rAlydV3AekIL?dHBLNtrYsZ6brYo$tcwN&AfqPjY;38M0vcW zXIxl?q$&7i0kaRY!CPtB%Fh)n8ScfL;$F~^T~nOwg&iBV(;*%go2YzPZAYYvGE3mngq42Kz@WAPiYcQP~pzx%hT$p{pU9R=$%fxGx}N{vQO^GtTu8PoyZR=Br9A` ztq{GBgh3F+OrVI5&7SK^RWTO-{pnE>$ug@&ty2ea58y9BpaFZrMvDzrI_c?P4tqahV1PQ zY13Ncg!MaF-g22l=7VtLvBR>cmI#cHeX97J1?*?fp^Ny7I#$_o)|YeW3a+zj#&%n4 zIJFwKTaDNm6jV6?^UY}g_bkJ!<{1!x)cmDf+nOwdij1#6Oe6yvQ-*-A-%dA8z{`ok z(K(vrJmHJ74K7e(Z1`}lnYHu2q~4R|96Bpc#h$xRTb;-xQ+jq(KBO9u$7TzkW!@N9 zYiYg0lN2n|-DXT@xjcI>%j@c%jqsV~9`qaXEIfC!=|}NL65H_wQSu2*jyW~DUG~fI z#hNkc5R3j4Uf6W=cfWm2k@Vj+Q1*z1FFG+~=h7s#KBNt*;4w@RQZJT)=1=1yN`ZB` zI;kfTl5@9h@#mR;H-s#OciXLOw$gA7Z{)Iueq%=rPBm(7l}Nwk#j@xOt>=J8wSBK3 zQuWUEvR&-Lir8xMYGClQO!gF^m5w01_xGya8(lqsI#k5$ub0G57ZTP!YY?+XTzx1=TS+qFoaP3Q>?kiZ_n%j+4>k86 zR;(X%&Ks7`I-YIoGW5Mms}xHa&6uil)S(@4)+NJp2RmTeW$Vsz>N>_waOTy6#4xkbyKP^tA0{LZw}O!EE7E}LrhdxCRHhMUB>IY%W316YrLdM z(Pex(#WqcU8s|pyA$7R`MRATj(V1Adj>84g>_H^-F-ol`09j)b9P!~a`~h>II5w?9 zqB8GtEgS&=jVxxrW42R^fQ~;!rSgkwUlEZhcbPFR-j6^qR{*%>LUZAvlJ|-QBE$?e z{>lhbW47aHD?M5E3vG+rAo}WY6Nn-7cXW>}ZUY?x92w}@K_~n?CFYQT6yAvS3g4Cm zfG}Y)pX^4fyDDGrE{(hOFzAFsJN>YL=%-?E97JHb;4)Z|>8*o|&48o1uH&S8hrrq@ zcJ>zFh_-^=a3lo=mU9>h&qU~F#W57eG$d@%p*raE?5HEC1J4%gQ;=}nP}$Mh#_3w`Bc~c#T(F7EVuefD zhda~o`sa@!*Xoz*UoaqWFb*4%J9hfDmU{-zYm@6pB*g>Uowz)q|^S{7SgFtEpq=ZT>?a587~g*#j$H&n+5jW`X#)3AE-sBhGxY$(^c3^*Z4Fayh zJc|>v-BW(gl4ZWLaMM9o z(mGH_u+~7=@q}BIz@p!f$u!+E;fU>l!NNobzIhKbb%1p!fp?@Skp$8Cm&JF|b?7f+ z++@ezeY*Ie&K=}i^N&Jq1WQWaOxC&QPLf~pW1D87jj~O{dP}I&o|eU?KPK?>oHXb% zwkrLiQB^gdUz3e-O>4kFgJPY$=+T;?gPdI(cF>v3a=kGT3(SO zX+?MEuuF4P$jK2%U))iF66x4`LPoeH@6I~M`R$_$qDf|=?({QhFwzJTY43>G-Lw5hrr5C2Z##fyI57*!$hTvG z<{=S9yl5=gsNxTqJDMXtUkqIcV1q- z@B~ySrLjA1yu_cgb?w}X?bl5$t(&<4N7*7_2e_48500M=j>C8->mF)?pQk42_)L7j z6+657GD(zUiJk?3g8t3ubzO^y)(gOiJ^|sLzph3KQ(pjF&hgTBUV`#*Q!*?qgt8*EY^NpuVtF zMSzCEqKE$#zO!FZF>cU&Qp7K1m zIm2F?N(iTVw`+aGK<53Yq3C4;JZ=s{f8aH?QvlZc4w8{2W!Wt;p zWa~^tb2?q?cxdJBHpP}E^822?zEi<835rJr7xe@{Wz9~oi8L^^0zUB$#EH-=j$+%LR3nm9= z($%Ogmz~4dx2}E8_(`qSVeX`8GuGpeBMxe%#b;ZR72Iz>IY006K#H1C&gukv@2TII z-&aj8D4E^%H|MZ8BgeR9V{sj+kdo zZ1cM8B5CHt`)RVDB*$}she|BI!&{TilcK>N zsI?bX1}W3sQrMmtzoa5t{zeyClSQl&Sf0~3PH@saFR33(^!!C2zL0i!9!}ET-61ad zOBk91$6~hjV^3^3`X}c-d@Z-w!Gh!$ zw0co@T?mCf#f^?H89*(VX`T97;wnxvH;R!wE8C0rFWj#1X?V#>7WX`KAuitn>)hS5 z61V=zgy{Hz{lwz6e5Hzi((gPd+lJB_$33S+F6CsaSCgke~o zW18RUu;T8EbxM2#h~jGlUtm`)^pV-g1$-7&))5Dj1J^-)x^#)F5Sx&?vxiu`+c;iU z9{7`IY|*B&<={JpWh3XZX&#dp?^7- z=8ydtw7xf;N9qryu7vU{-fZ+gyg%Le5BN454ZxkKcg2d7_`x_3jxeO$HHCgLc4$j{ z2sPG2UQQV9H|8rk?`BB6ZJ2Q)^pggP)! z0kIRh>ODJmNGt$P6r|dysr3Qe-C9cg2?4U1Y_2fK@_nX_GT8F{3~w|30M2bsyNAWm z;y7_Zl0U*%IAa|yYDYP}=KSZ;I&y*L9aGWRQ**j6zXtYv(ib{^AZzJWe_KzNtktx)laUAfny;tnzS4)(?+9cj(CDu3U>w+<uG+l%+!RO}g4>?|wZIw3;{=4s1vZ&~kvV>SgCIno}Gl}On zZ?JLs$l(mc$n5B%wl{=C_P`8IOm4V_v2t*ztFV~UEIDN1ZbuE*&BlA83by{Om zJSKIi3Dl59;)flHfnLY>0b80A?40Fy>~luw`(h;b zCuOe_(zlig4r~W(v3IOI@q1~9%*u(Itm_;AIf-9i^PH)?*~1QM9X;sk*~MI*s60EY z+5GqnMs#O=Qf8x<|HKLjg1G%I>v=97y9aK7fYn2#!{Kv4L)%r&dt*$|WXyVrvHi^7 zfAwblP6alk&9m@=DP~YD2LFettSIE^r)w zT=mQh^_{_F1&cm+`@fXDnHDS&m)Y7WSE!7)1K8M8`J+<>)>1(+cO=`9c!k~>hJ>Yk%Hlr)h6&X+Dvz~tLAr)Di$sEV^r{gS0##yw<8?Q%c_>T zC&`9Z*$*0Fr3R7`RlrwybAQPGg~c4J=sc$!cD*C*ubiH)_OP79TfPDDhO<)G*9RuJ zUHOU_U#@}ET!{g z)QgG_r3hMW5!@k7B0$ZTkVUa%F+ylzDIFbiaFWV* zI##%iY6l#N<{HCr_06C##gRZUO;UNTlDCYWljLcRa_&U=xx(qM<;K^|(!ZE%5z7R* z)`L1ph}uz-q=PUD8b>?g7qWBfD#SmZ>hK&@%c2xBr{(^(XPLYvxKDgMT4_*@HFfE{a z_FR^2`CPAGc2=COI|+=O1hH=Oou(3MI^M7NO77shDl=T?QmZ8-iF`4QIC z7dlttWjl?pUt-$~Rqio=@SX6`p-=Rv>8xxC$+ac3FiCvbOqp<4<6+Kh*B#;DB17%; zb*h6L8I&Ho(2>Z=%(e9PzgPl$uoT_zpVF9=CaEcMBS|~QndwNL7pZTn=O+)Gn(>eU z&&qQ=dR%=a<=6;9!cs~Rqa#!IU$s!lhhQbWbp)1xxQ2dL_O6T!v9gstf6PjF^l6I@ zzkki6W_rM2noOHO;#gNkoR`(@K+8maPFgBweSzXHAiO2bHrdwqqd4$ z0#mkS2T_D|RRh^##{+y+ypss%J^(Z437IVY))Wy;`S~)X6TCkK>?yu1zPGnOMtRe> z-$}<^l;wVg6*K`Un;-uyv*7feFKEm|tVj;l!EA~QyDOqQY@ePvY@#DC5;~s=4WcdR ziZl7HAXW9Umv1pa`(*ErN|vH}v%Sl?S@L{-YjkJJrOzHR`_sL`&m5PKRu}|AI@+4K zyrgkt96Y=<^#3aS!14ASF2g-~qQ=GkVD?Qp`oep1`~%K=1sKeIX2=u`Ep5T+i39yY z=-b=U`kqyXc%l2ouHSpyOCK;Cy!Cu3M#8%;uU#D@w&>EkZp7)lY1_bt5~B9?Hxt2` zMX$g1BcVBN^2c<115S(z`|dpAy-p`JS>O#mMNFk1G%bbtPan(XEvZ+B#r`xgqFMZr zk5;!7N_Xz}^EXet4>o?}gLh74KW}A@pX*SHN1NC_LUusHmkd?FCICo3C4P zQ6(*iGz(I0liLK*&N3l)%>q}W;fsWYj}(rjZ)s{D$BVCo1jo%x2pELC+6Ed5sB1CR zMN4hrE89=*mBoK->BTp;B3-Y-or^1Z4A=wX(Ro`W_%64 zdJ(~MHtxq)I?q;5yH~>065Sg#p;lE*`|(Xvw$+Kk(`-p@iV!i$F%sc9^x;(4VB*on zq?e@eh}_bzE9~fV{0*Gc5*WkO*T$C!)#veY0r}PHO57bE`Q4o1Z!!83qos*O<>h*s zx^U-pket;g?ATmRoUvLS6IEWdzTXt>EjpdgaKia~=W~FQqV$?Ii|arqU4BfckaI%? zJIZE2x2_o0BDJN;lX51_-A=4cThq?niG=@fc-$y_ICj1^+S0>>J0885An(dcjhFo*!%8n*dim(R|E zsK6sNpisaZIm}ztFo}XOMTmE4F>fwau6~SyYn2a%kfiK(G$X2Z*ad%89geXpL#g_) z;rv71wSQ^Qlgte5M@IrGVfxfD+nzvEiV)jCTL4^@>GnEw`;tBSX7qLAvu{fffvF2f z@aqv~rs5)p6`jS7CGP9-{-}6XE2EwXE&chT;23Y_jc`$3BE))4N+|*L4@bDp&yYTGiO73D~%L=h5h9S$O%3?a+e#4;V? z!}D|S9^~L+OHuQ7yw^y-^y%o~8`j5laYx&g=Q_^K7k>UVr$4{F`11s9M>!)i6sukM z9-K3_k{jx$(et)VvP@l-+>Z3^I%HwCkZpD7+7${@qPft0E9TAp(C1ulb)72(maopm z(i`5Pu5NjGV_w#{-Vw3Rek%9xea)%^=5uvY{@mzuy%n4ZGhOS`l8p{@23J#a2=nq^ z?WQB6UQPbb(DzkAY-3kA=Kf;ZS2$Vb0(%s%A0^QHNfQO!FEOvKRN;zv&6g|lkE(HVnIXTe zIzi+%eZOlsAWIJT+_Mcae1&E#>$bKf@;x(Lab}Zq%P+2 zbAiwFAbQy&L~duC9??W)86@g4JkcakWJnrh$3D=shNU-&lJb}$*BjO3JPFqXNX%eY zAUWY~qLgpck|gCxJ#RbVWe})|)?n4fn$Ls`SD`XoMx$EzgY$D0wC8h;jK!VtXYzTd z*)=(Ibxrb$$mdg9s!%i3SVI$05Z$Is3yso8q*K^3J1KIT!)j{{J>$A*=?-1noCFIw z3rF8-hqOOee0j?4F(Prmm)CD7V2&%ROLqufo^P8=&+W0Lo1a&=@t6XhI9uS$=P@q> zPu7PSd3}dM(|L0FvhtQcTP;S5+PGhfJ6$Zp^0L$D&L>3)7NV8yfj%l>xe^~n!k#-P zKn~Z2U0U?5wo^F>ae{Aec&XZX0w1>I2e7ovx+{i|f9&LLTf)HXN+qzr=Y|5N{b*kX zJkg!JRm*#rwR7x%l$cvJ@%AhzSUET2GcELpT~{j~`_*`V>?BEw9jRqwZnKDTtX@|j zw^0@COy=@#MR@krmGnY!sP6z4RPMqdY}l$fwVMC{e%L}N+0P=^?}{d8d%yG2#H8!n zpTk$33r1JYNFD&MSk1FG-*ibGtc$R!_WGoY_P$@qGjlisFN5LahXWG~31rhc;nKKh z^vdVclDW?%pUF-{!+xTbWe@ z0&u4hB~2&vCxW`?d|Wu~;EWtKO4+S2;{w*JMO$%_JGSWxBI5Ayab)H5YDkWvqt4lDsUl&k8rB+=7P-0Sb<$@N-Jt|~A=--=&YotctZM>&j!os(=K1I%I$*Ez(8g=O8h}ThnTU!>_~$Z`gv~)D|zC zF#lLzZ~(UZ$C(cP7U~XP(cfDaH}Z35b_d)5#ix9_^ULi_frt4Y2LI{K&zoA@oKc*x zSbsIf6@YRFqkoCDeIgL=Pr_)L*s7+BFFBp1vG^ib70rp2_LBHWl{7#CJ?&bS*Nca_ zCjyq;9!P0TGSffB#(f6jkaXeSv0g;2NAw*gnnBcblnLZ#yBy_I8f$UL>83IvZr+Xj z7OUc9zG;hVxwO#QdCBDr*3RNeeLIP@H?!H3?ffOR3nrp+93^dWVri8i9eYE6;+=G5 zVTTFOIx3*Pixx=JJ=Xt>OiSa6iG5gS4yjG{vbv2pnWCD9oTiG$2U`8AO`Aiwg*lr9!{t z_MiV^6S~dv#XAlbZdGqX{3~9Fw9SZ4@7Ef@^2>dc_wBIhneFH@*yS z~vA;O?0F|4d5|yq19D(=VBzRvB~P2Yo9auYst^=!*PLT`faHu1GNPL z%mx#?P><^q-PGm4rexAvY?0qaa3&7Je^e{}9Q-(5uOMA53vt+PX0ah;DdbWHbNaGn z>e{z5{1j*;3o!~Q#J?6ZCHH&I7$@fL?5hU%JtIyR{CQ{={wk*NUZa1}i4`uZy6;zG zw$^UG07m8T;s@|y8$YB8R2J-X_zzDC`1t8)00LR{zG<%VO<>z2a^viqrn0 zv)P-5Eaw8lqiTOse~vSW6qZV}Ss*`q9bzVXvc|@nq4{4IpR7+>R~^0X*Bvv-T*9u+ zw91(=|LUA+$c{08ufVw!{092sDsCdrXW#V)~jYZ`lbvW#Z`z89NBqb`2%8H+uHAv7M?AuEJ%x=f<;TQ){MMshXG8* ze^ta&K&Wz$j;G94)En6V-Nm+k#hFdjGxq%0-sP<+iLx3-Sxl-Sd{xyx zkehIT^!$TFe&PI0n8Oka@khU0!!K8cw@fz}XqUcEX4o+g?kUqx7BvofbUBbW!#d1$ zmNW*OZLoOw2|fFdUnM`{MG%&zy~0!;GYP>B&3G!o=-Iwm=;H7sdb*tYie=BZ#X&#{ z2e7Y04|%1W+{`XB^Lj^9x4y^c?OvWFp%fh}?xX1E)U)#M?n3ssM-T`G)6xMb?p>!q z=Y~O0$!79%Yh~Q3zgOb5Zo1>4&`7EZJ&GCgv>Tkxjcj6@%UfDKXS~;AQd}){`++VV8+<3?BwsyBJz=XE&p*X4U7H4E z=5!|JrMEypN4N1V9y@l(a7Wb53MT%dQQ6<^tpT`nysmd!LkN&N8cU~^jjmYMn=2zg zY?hLl8K8rUUmr5_Nhu%QV~0<#1r+XH{qof-r^@2w*{QDDAOa7|^9n;cyCep2Z(0^9 zrf%k3m`s2Rm^jsqz4ZJbM(y%OR2(pKt7@JpBxE5TNm#@5x^&v+|$*UW4MVG@_jxb@Q8W z%1%&w!}qS$95_^wOMd(Mm|U#LYRtI&V~bC{udL8R-gFota>b!)+BUC*O7cPXvcub4 z5v(E6zOC^x+?-82r-ga&w0JJW&ca`bw|72n7`16Ve89QrZLBOh(Y>d2o!s>yV=GES zF5}92H|PD{<~dZ8^;*FK!0`E~7aPLNf58RE;9bPK^82A@5%1z)azUak3Jdo3R~&m> z9vW>&LM10ac?)7@R1x9gpdQ#2-4WnnT_YXU`@Gt%4M^$2hHyF#?oW-<+7w>rtAo2? z`1`Gu7D5*cfi!!)np%a}Ct;(Cv4Ez#Q(NOgi{ta@co&9T3P+tRXRjk6{Z!qdEh2&T zg<4vTKUeAaf_=2g7NDv&-GnuVok)iUQd5=!3G0r=tv!as@|V?qy8-KF22rc3Mq#EO zOS2zQRSzz3lpfZ10V1Z>H%2dIcVAQT+g;`4pG^Y0zbKVm{1&zx=ZxOC&3+kfIVT42 z7XFkYic`!lQlZXEa;dK}p;e_(=-v8eOxfLZKVIjvlw}F5;Vj7pY+0P>0JmJ2KngcV&1eR#+4RgY)sS5l4Tr z;IA2R_D5^4P0Y!N0_Biz#O(U%ef~6{H%6=j^EA+!Q<@^g^i=GU0fs{z!((za$8B?| zvYR_bWBUK%>n(%gXuh!T0KqK;xN(QzzPKe2oZ#;6wk)m*8VHNK!$NS^#e%yN+}#}( zC(r)Bz4bm-&zJ71sgIRrHj=lgxeas%5i@nZC0D_Pi5A!6Vb}F)~!;uXRR0!sfC*m-g83>_a_@d zAy9H>nd=f`6g(F+=jUrBoxNet+^X1V11w?HY+x+}1W`-p^TJhrmQ(!6DmB=7xq4Fv zB}Htm7nSeKLSY!(rg-0~$FVbwj_MHO0bFfr4Oq?h;jz`MDipfPS&rfANv*~}?61SfIUwrOX?mi=$|Gqsj8yE_YDq%`g# zCKYzK&5;x=Lvk24zh2LX*`j8Jeoj~+s9JTor4)89Wysw3-{@+n`iz_jy5)~FMJvR+ zmf(&Fen^NN;q@(DWq3@d-}7u$!${D}p=@#qXJBux=|nAJV^?95P1M)yi!|%Q3qz(o74CZaeP`ZQ9{@ ziH@>e5>3(hFN?wI6pVAw{nAKvR3hNuK)?!ys)%AlB;RJ<5N&uTdeT&?D;ta zMkvJ{ruXKGFFr%Vw~ZLRoQtf$-{dw?L>qibFkX^xg!Car-~K!``j~Td#DY#uUe%UZ zt38nBF zW~eiWaRPUF<9B+p`DT4^s}K9Kg+O0$1`d_Ax^UcK))D;n8Q#P;#FinWW^eN1i+bOR zAl$$s795cd4x|1p8^iLcn);9(*d4z}()YoZh}|_TOZrKke=tLDFn$C(e~!d8(E6b_ z9ka7mcM_PiK@jybPc3k^!Pw)O?L%F9b!)xEW!ufK-RUkBy@GpW2li}0Q=MpCIuwGz0-Wt^ zx@jk02Ic@-kN7Kp{bCbx7&M?zwj*VjG~{Z7(%b3{6NYD(PNikus5b-Y3oAFohMm4d zDrFg}0P}N?Wk8SH!5#ToNoKz|Etd*iqcHbAh6WE@Qu+n8NK1-KfTR<*%dAhwkadLH zsG1{Sbh5fJH*@TcJ*KBBX-3&t#lV45TZO z<5+YiNQRMAT@?A(ZLf4jdCD#Ya~a`elqgIuPyec7=w@c9jB+R_ozUl0tfDX=+#SQl z2Yc%y1AAkuXzK5QX{}IOqvHL-IFn53%Rmm0)PWtS=#H=>Zpo-PbJ;$`b=S zcASW{4xv@PgAtah%|S90 z{r)?-E?X8LC{C;G1a=`>&2UE5eV!f~3@wFNd|l zG0>o(&gM#hl(;Tf#_kl2&sFQ4-(HROa!K!Ax-y^;&nS?14m9J>5HQxN20I&$K0fMw zZsFlxW(QEKbKH-dWkW^o?FYo@1}8As#kMpmcwOh4~=)B1AC zyAs%kMXG|CdIar~ppxN^6sc`tTz`t|JG_??|I{oU1cqmO0z;~a zTGxHrS1*!0m|=@K1nPpUfJ)CD@At*b_K=f0}|8PNlc82LbahNe^fKaO+NIjH`a00 zox_UzCI#S<7hyoP4dWnKh7oDdyXDy(+WtEB~O9KdHrL@wu~#7S-CCv2TF8 z*Ubp1U3Nw;&{T|Cch0nAd*KF zoZogT+hb#tjQ$O>`^$=#hj36c%QUIfw}3upK_;tE-S;jZ#F4xmTK9E4mle0p{zC{{ z7o5v%vX5{91r!R#vn<8}liaBUFG1vc|0H``Kdv@cj1`ILknjW7?cCWvCKww! zx$fxsG=EoT;+TwA0#6?g*WWr%`}9vg5Bj7xgXu2^KdzOp9$CMlmN;-#UxQtI>>T|^ zWh}N0j7hVnb#%t*ZL&1A*&kA6Am3d4NBLt~&s8bi+#Y}M^uq0QCA`6l$tRam^RpGA zt>Z=FnYGKGT>vvuYn);+FOIru=>2=B&gl*{-J$z0=T$kqIG5%U-UXzOjb0x*{#h5PJO}r?`PH8NB0a%& zTpLiMp01N{U;v)hi9_XcYx{1aHFtD_|7Ky>scd2^^}d> zBA4iIww~7d7RhB&oJtPEZRmTb@G^SUsk; zio=qH4;{_^Y*FJ#yoQGe4obUOSvM|;@+REsWc(h8S99`wP@wrlL&rTfas;a*b{a{Z zJ+sQMzOTq~cA+U-og6uuFq{g7wnPOOW>pNvNZ4e#i2$9%>`E#AANy$d&>`M(t+1*4NC`F_J%S1&y*?ur9j3>BJk^vnfHn=I%Zqdy|k!E zcNjgdG8Bp_kI?v9V~>8B)yBT|*DM~(fdR>I>?NLZy82{O*1kPjrF1(#2^3F@Cp)~( zM~@Vdvc5WrD83jO=3`et9sz)_v`4lgqB(qak}>(K&NX39)l}UpVfHZ7Wb<@f`S=yC+(!1Tq7R9LRMV{?Hu7A_OQzZnppWXdjzP?Gl z3m@G`P5LQN%Ic`qE?-?&%=f{v&0xKpSv|r{%{@!qJNlSrcI&{xk`@KTD*{rgfL}Hq z$~JrNWaLPX`cV2zrr^TxMlqoC{O5BP_+(NYba#SB1g!{sHS)cnSwQX?S!*~fny&rR zCO@qR87+Y5*XOAP>_%iL5_Ltf4zx~}w5&I>Nc{nr7+Wiwt;i*{omNd8u)UT}z-Mvk zZI65IIz1V1#iQXe+n=KQCg5L?Yw9}afyPJ>PVZo~k&!roN)JpcoMUk&OJc|FGP0p8 z%dEe5jifA)QtWD(Iv z*KP;lz}2^)Q;`8Dnuv$aUBN^9#EA<<1bQe8=sL*?I@|dxKCz~3*?Byu{Yv}iLT)lQ zX+yKj1^<+a&{TfM&#t!t2^L;?a9b>W6S5(fVdOD(xaN9>~Mqhxt}f6b~22I zp#FPk$K({4RTh_@w21tG(9Cty%nQ5=evl zJd2ps?ncCC<29vJ;U7i$OhI8XbC>FM3nO_O_WB#H)P7Ndo$kG%hN^$r4)J9A7a%*n z*aM4McKvxz(Fhs45V4ny+wl4u&F*+n5`yl%t|sF9)(OF3+4Pb9ri=Wd>PBw#bGr&t zT#50;fXF{nu>)fJ9TWrEn9kc*KMAH4+3@hIf)N|>(-ug`q>3$gM{_*0nXcC5PKav+ z#xAKePcupUUHdghf{LP~vaVK`M$`nd(3|g+1Dbu!F1>DKF<)9qoONd~~8Rv@=m@W|2Hv&UWB>=^SoQr>=={yFy=S;~<>#KeQijKF9lZ2rc_~i-p zA@enQMTq=!WsY$oPe&uX`0ol9HkjG59!MteT|fl&zTshfU!8oS^?J=_t}_hn0rd-H zRBh_bbtbtN)Ztg+47i5dn=rv+pkY<2dj#RFA|J;2OoB3Vue$58dU_@Izt%dP@ocJK z_}Ys3K8Iuw)<=|eaQLoR<7ycg7mT@ZN>UH`^BRJuSDTYoI<37vwGx9J0tK<8GDYNf zd&9=qg`#m_CK9*#?icvd+S+&k69#`9FLE1l=Of=4+o{h6ONvbeKYmy7`pG-xXLsXQ z;lhHZADo+?a{oS1a<)UR(t)wxc6fIvGY4#B8BM?!*t(0Pxg_ID0CACoJ9~f6!i%|w zQ`?2|t0lDwot=6*_v(RzuZ&dS$w*Q09DLrxdLQMn0GZud`5cCCCs8(gGUu_LcDcye z3(=y^C+b!EZ70zkp7TBQVE_5y&z&v2;{%hjzzLGQhDieBc#^%qgU~s8@z%=xf4e#V+^|SE> zNb&sqMYxt%ddrq?>oOPOJQ@?$CMkRuuj?|OLo8pGgr?uzrx<8b)!JE8Kzht_6M{e4 z&;#8I0*ug~n;yQPhDJ88*(SyC{pY-(=ciIAg7S~eR|~`(;ALok*6L0Sdrb8J=TAs_ zxwW_H!q3nMN!VAszx^2NCy1(eNR#B~>M9~#6oe`>z$PMUBUsl%({sb=`@YnOYDIqc zU-H{aP)iGs#{Zq5{r{=j^Z!isA^b^vz+tiJq=jT6RCbe%*uN#5C?iI{? z{wO~#T>QIQ`G*Ef33|AD{`Y!T{l{Rp$Z?nEHEcNdPt_i;Tbal==HTCZDyLSb01u1| zO7U+~NkIvKJ$%W@K|!^t=-XkdCX1{f0zl$^2Jjp}qe!Z+lRdcXAZ({TluKYPv0oPa zX+VYjF|FKFUPD4`V0)O0LEmqDEm1|`HBnYE$6x(7bVl&r0WCukbmw{=whR5O5-H@% zqtqC1j5coM2f?IkdgV;84D7*LvVz{L43|PrqaW;O$k}wDMj^U6d{EHzbQ+Y3w!*xP zCq^dsAf9N)Qkayz=+Mb72B5<}z9TgFm~Q16dyqB2IKF#kJ8?GaHk^Y7LeIr3@Z&jz z1@u8^a6wfK1`w+Y(_NXFQlFNcso}JGXav_R<*SrRIJxY& z7Ik-gM>bQkch&ar2NhjAz=G23Uh%9pTE?Y-ycDQWwVj1%zEBh}`*Cf>B8k;py zS@(Q=1*RZtaeEY)a+nl&I6&yK;Xp4$^e@p6$RGYdk14cZD1Iv+c`g_>KerF{yn%IB6tU1qu*c>C56CsO{UD z{ViqR=|-i@vDMM*w<7Mw<@s4m2MMO`etoA3RbjoFeqqn}P9LFr3-Y4P1fIX7f8O=C ze#2W59$hY$mOTeBiZ`@FhfbIP2r z{}I?Vce)??kb?+RMj~?tr;efSKyP(UdvYsj_XN|J^D3LUH0gIoi@Kx6wM)a6_l1Cta%6UB^V28Eo3P3k&$UM*RlxM$@ zSdC@*U51$3yKyjkMud#<_GiUZz%}>Jnwu3GN)HEGBX^#6I4Q;c-nM$d`*YzJ71?RD zmSp1lQli(q-TeMe!}uT;_gM|`ef2=p#mf_|+QA>#xi_^%*KZ6@^zmmq&dkvdq5+;L z>|(K}Q8t0qK>P(4NPbangkA=!P*<;7dGj^-=GM@G zX)DIwUYhgT1`pnLjVbe3M(;-V7H$2f==i$eA!F&^fUnWk7tQS6#?k>!bEYQ*bv&3yS0tlJLK zrCsIe?HDTT33PMp7a?Q7E{L4lBcTl0Yx5cKxgnM#9mdUyh8K0nIgpH<9D=l5(`2XL zuHBF;JWlOsEJUpBryJ)|+cpe--_PjY_$BXF72tbPR`|ioSGEJXiA`MT>B{cQ4pBio29*aeGstaPc+2F*d2AO=dU=+%$33b$^FMhn;fgv0+-;jitYm$>@96)U`-wZI zAq!t`Q>fLZf}tXy#l>aR@<-}sRmYz4#_yY)Px;i{4pVLARNZzTjzs-rAzB|N-BcHM z0K(=*{;frAp*diB+a$E3C2DSducOOPV7#@n0y=&NCr*{%vp(;+f*>x$@H@3%?nlSU zc^%}1P)^Rl8fO7N%9PZde6?jU$$*yx(J{GZM7HmHd&KjVmJ3qsjFUz)U3Z%}^?L7m zm%n&ZO2({ZVrWZd-DD|}_M=aC=^6!ojP$v*u((OgUmHBVPLzlK+#!~=TfHSNwvxv> zabwK-^}l|==`GT1?f2iOGeFJ3@tr4W+CkD=xaR%lML)HbkI;YNcZiNS7&k-3=+$2W z^;(^Va_QdOne+edUly1AF&<^Z6$ubFS40Ji{)kt$!@exAnSG`>N{R`d^ybfQj{p1P z!zg#^>Xlq>7>n&+!?69bvt8L3jZa3mi82z(w>URvqrZv7__xztO!zwvyNE42116T? zwTFH#xf>!yOg+Nj$&KF>8M)+1_e3a0f7t$t>l9B#nzkn43MHIrmQEXz)TMcK;@ImM z(vjOp|HY=|Nw72{Z&{+;R4G9E$RCf#y(5(D%c;N+#`4W&L1P^LDe=;P5eetMHCLASo!=af;S1DTuZd+7KjJt@zA1Vg*?_sEK2* zb96TlXMm}gVuiEkCX9s5*6bnu`+9QVz(i4dsq$5x?iIs5aEQ$@2kVC|adc13qlq2Z->(}lGJi%+1+&M|A-(7O={Wi7^@@`1i8gVZ{EM(V6vIpCV z@4+y$_k5{!l}F%_Z=#bkGK~GdQLjY*eVh3#e$Hfh#&HqKzNcV6eyzO7n4I*Y@)4ju zLdaO*dB3mOt_sQ#Rm4T>iv2hQ^Ivd%ga_2- zktkk2*%h`Shyts2V2A9a~8gy;ES zI&(CKjbHCHEe^!npDj3TT_!#|847(F_9oNuB&Zp~%-)I)+378QMfX$0gb?yYqx^TE zGUDM~jH(~x)GJ1!dc^4f&xRdq=Cg?6LJh@FY%3yJFS8n7G~?)NFgRZ49>b#XywnEo-YDVPFmHRJqgtY5g!g7n7=q`;eQ#s@QN9Dlw_`$F z&0|p|Sl$bN-~h#)8H?F)NxF`dKb}i#VY`uG%)x{~Drni^#q9Q-yM5p4W)<&UaQeEKj>x z0-E1KyOM~dOSgnxMKzqwKiP3es;MxBVa+FG7@s`)GVegv<4)jARbT^|n-kTaWtX8w zI{1hKflR*O*uzLVY$8H3Ito)Atjat*Iw_v1k_J{x-cXYVs{M?hT@IZ_o7NvbC3Lt$ zyUrHXGn+A2v3gvEm)%@euDEvbkyCz>-0GHE`*)qbQ~aJ1PMs`ve_>m1L%LSeVdzAEOW3t+(^@pvllUh#0(vK0N*6XTEL{qJ}U#9hlc@-hNE6n8#xH~) z5)}^0jYPuVqW@01ir?^H7RYQvvnJbT_-Zb0{COHEBy3pwgA{eT z%_1SY22uo5?~!#+n(t)D;A`=@XEIw8ig~<6hlc4A`m6StowICHe;<}k@6BC@GlJOF zC}2=Z6Hs$CIfY>`K3ConGvX*m)7y{M4}4m=spJ;enGKzJ^>+~+RoPsi+Z{_j_YW`- zKjqu@4|v{V-1TL__XP3mWf2wN>ah-;_^tKI(Xf3W+gqm+TCHv~<@dQ;B9&kLxp6k4KozwHc zwa=YQwv==$Ds7pkB5eO7z}~1U??BKfXXT8tHg}GZVNH){V&L)1^m@kUl$d|{{-(#c z;IZB^LDKPmOmvsa4xr>v6rD}c>b41eDk3&R3T~0l>FX(nC`_Zd7&a-XuGSApCig*1 zc@3`@Ml$GSS(LwGZW*je+(Vrt9z5P=n1*PVV312sF#wNlTHHfW3&jgJz*Bm7O}RPL2r*31)WJdAinDsflvwoe;n zPUOYXG$_9AeyU^~!pI*J^7VMS_EyVilV-Zsk2)w{)u~qweFlk)ksu#T<^w+bVU*q+ z<1XhzhV*66iQHrL`-_;JyE)Ie)_`k~6eru-;^9@+yH?AE4H-J59F}@K#yz7U8%mO#?NWKg&Fj z*=p)NxC%oC#0P|Gno@$YQx(m~KJ8!I#c@;&sA+Y}i97bM7+|=ZkEFvR8j&75zGrmd zhNd`{%NGdd=SDb?iW={3;MtjE#zpXs@U~q88$4L$Gi}Jwx)RpnPxAPo{`vvuvy*|_ z0`ddv!cAQ>X3p}+UArSUBuB^MrpuT}%Cg_MUDDD*e8a_#6roGE|4#kKyt3{P6^hex%oL?2#J@#q&e7U@5#V66migj1(qk@e` z$Kcak&YceuF9pd&U{T33&)2^J2)xZu%a`=d_l*Mg4kMQf>Z>+L%{E11yEm)TvwDT3{qr4 z?01Pd_eLu+mPW2uLeHI$V;#k{xp$@(cY5%-tA+6M^m+$K4x{Iwosh+XKX&qGX=^!c zBd0d7O;f)5Vfd)Mh#PJeG+g^UL$em=T;|tCdK6BVEuc3($2(1bZ?Wv^s}KAW>=dqo zJ)TOFP5Hz{s1i<5Y}40_+iVwcJ7tU$>C$AWk^fs|xy-t0ddRJVOaA?MxjfH^P-+9? z5r~k4>v`%>vsWG-fXS`IjvC^y={p^WjgFlROuq|T zKy+utjDx;gjE>HNku|ADZXxtF-r^)>of_d*8mq;qz%pnw_e~3^AV1INm&Tpms>MiT zIMl<7<0*&l3j|AUH_!9_Hwb5or-3}fvzw5dd)A@G9${&eg~p`u`>9njakXf#6M1P% zVgcRY^i`2aF_295i{7!;&d`5m%$q*$7iGOQkqI&+@TrQ2$0_HEe9Rxd_~4AcN-X!= zHCwD8PU2laBXV1c$HmJ0xSC4hx=n~*f1eFbaqom@-Y!-@NyCMtccu+rUG^O7xE?$q z@$nUiph`^@;l?gQ#dVo?y^8+@!Eo%Ij(Cy5%7c(n-`lX&co{Eab%w0?YB?|+ zh=$Hn%3AGC?Ud4wz1H%QTBWN27QyC84#`)6cx%{P*OnqC~`GHL9gB7&7zs*!+ zTls2Nm0yK@ga*`n-M*6G@lTa`2;w1)8gje;Pd0lxUYT1yO#AcX*@%M`!Jp#na-(E| z%9pxJ(hV+ZUIA$>(p5rFXz2vP+@c{sTQr3RDe()04{!QQFQ1KeB_(38@ZQA($|~fa zf#jQ0e+60INn!}3GBz%R)s>eYUh&6Zd^cp!q)d2_!plv%`BnJRHuH~(kB;z!!s5k3 zuAdW{_{qR(@w!GvC*xEK!s?X|6D&WN_4QP36X3#t-?yS>TF&G@i!i}LfhLbpO47&} z%)L&E+4a}k;b_U#2Q_SORn=ziZJ!x~X0zQN-)_n%8fxjzDmhi*-<5gm{M-+gk)9qw z=33oPm}k;2sW{w}g7}`~W&AgYdD;jG!$2ZDW8dpTM|l-FdE%ddi{#TrfBR)7N_%ky z7SM-9#LJ<|!uOKN5Of{mGFjeRLC#40^3by;4l9NNd6oN=7&!yW+q{ljLyivx_1C^T z>)B?<;NE6VX=g9v@AYm3-^Ljw%Es|bs|>y8#m9#-iKA(}_A}nS@?9Xhp;S<-0r@kS zZFGi^TZ&vel|K%{?P*}Vu_A2atfw{@(89oc0t!#n+H}_G#Z2X>$1rvG0R&x7J!639_-5)>H{r2?6#jSrjPL%gt#wCdgopmS54c76f*U zz#9tgsHnMd5l`EyERccD>C`IQgwx(O*M#9Chs()W^2K%2gyHv*s>OWfy}3M_7F5C% z7vi&|eEX?H2?j8TBOQ6g`6a*Jv(8@!tmC(UH|dZdL#-Xx!7}K>KD@H#r@~_)Ixwh(D1g6P8QT`k=4pj8<*r>$?NmW{NN1r2W2Id?)-|ny-uN)4pgQ}L= z*z|9{X`0=8o7Y6=>h_H?UiBN4e&bs7C3_YwAI{A5--Q&?z*{Jc0r zs*12n0bb8X0M_z;LcXx3kQuf&MD5OHAdJ%8TLu&xHvzN+(E{#GZ{g|N<61m4Bw~wW zQ|B6RPlg{CE>ql)&XYGRme@S zjxazD?)cQ&2QOZAe!;vPIt0}$BRU;D&yiBTV>w{uyh{_kNMUcv`oV3;2R?1CqVA1# zAaw$;SY)xa{%1r&EA4a^IkDRwYpX9&&nmVkH4!xRLtwzTBC=_k~&P_lK z*z%=QaHVcSvqWsdXKqE1WymwttWHIg(F7=&{aWT$OVJ&RlV=oO&r2AMT?KOHEZ=YI zEoYY`ricC zdKp_8a&K~97c+8{M$B*74Pm9D5dN zFpvHeu?!9!w`W$;!fI*_P<=i6e1=T#2oASO6kfn+IbsMVTQWOKp#F-I-=0Gl%ut$g z=OJWZL`VQTIxogk!~M!YJOB-E2&kE7XN(|b!|@OmRPY5~;0pwEqnMS;c9gLjcWkr8 zS|&1oquO+hKjyCqJ_cdm_I09KD-RW14#ERVqi(#%XYVr-`bdsL2F8kA-D7XIlE>dv z8+GC$RiJT;9>>EKIR80ls7KiONB`c5oE7}+ymlaWxzai*9>!#E!T_FWQFG#jn{!;+ zYD_W=egRVm#YE96iKX5qN{}}D@JzK=bBwR$GdClfZmYB?{LWKb6wnPc=b(`NTr{0% zV+28eDNMA{f}k%4c)0NBtYCGtyyOXB?Y!{^TAC9$tez{6wO#k|N6i%BO)`&@p<3Z< zGeE#HM>AJtmX_67inP%;Mrz7Fr^1X6%QI#H&o7#X+lm+4>8zZ17{3hGVX^ zmTlU0b*8qJ7brUzsGxUzR=5%&yjC2so>XpI+F>Avr7x|R4W!(AQ-lb=Z)$6eprasA z+Z&;PAv>0|bSH~s0S^N*@;xp>`GiTNrwk*O>{jk!eJ`f(MA4Xz);V3*Zj^OSUc~3d zGQjiC+Pn^*5~6tPGO(#nI4jz3`?o&?)L%K`TcdP6j}O%6a~ovMSwuKwUuigTWGH4c zh^PF%Y^nG9n3>laG+90cMSIr##1(y?*`G_?VqGWD>tc`H-Xf&L%LvHJ`q`4#l0Y2q>}yAyCy}YiQjt~WQJ6+cJ91zShLdh6{s z{E#<-8LtOBlZ~bhHjbv;_RNXoCBcosz+LUifGbiaBwCY4aoCX=lX_Z}eV;15^NtlK z1xlZ!zZjrSom}61Zh93LhXu2O1s9gm6RN94dCKl|STi*-S|x?{-OTFV3MI-GBO|xw z%~HMn=ZNe@Gui3C!Kr7H>o|fBhvht8Cs=jAB&g_JO6iIC4@hKu9^#p(Z3%EnMoR%$ zhZFdVw(G&~vKTL?7JU%Oz&!}@F|7^$en&$Lo7LmJF@1Y<$)~H%>0|(l(;)Hi@tv`6#H|AO7;crqeJf0Lo^Gut zA6DA>)8EUJdy^!7uU4Lsvm$Oy{tBgUidRN9A`ta_? zxN6RZd@f|`SERQs!STNHx4j{s0ai7C_&zon$g^vxAaY@UHd0CMq8rIfX^F2S6pexC zZ$^#_#z0)my@jLDl8tD2dcSr8XPBN&3ZAMLu$>DDiFwi;&V z8jxHW7*(~l6&nisFmLcru40~$%gQ2jDzu9&G+U(59l(%F7N75!uA2Ni*J{W!Tm;x` zIP>xu|F!B^W<=lcX)fP4ch&`P^0Tk|^T(;y9E+{kxRKv#0sF5#9vaOhHCVp2u#Ft#xM}-IoSW9pfPB z383P+;VQXP)}ZY1s;y*w&AF%X^+SQ#DRiHR&-_4rl9O~mISt!HpfURhV?lVKx$Yiw z`FZzA8^%4><;iXtDMsWv&G0CCz$vq)L?A_bS;hpnpmZX!vG^~D!(CjD*pU(c;nzL? zB#W2q(Lsmm{QJ(@e0rUUSInNKo5~;QsQ`=<$~i&(-ap)1xFQPOL1waidYP#zs}p2k zPV(}owiX{@hev++D)A%&M+L=3C_<@PKmM4hbU^R_ih)yag zDy#CyRZ-~+pB&c??pF@x(~plq4nd0FFQL?@`_4SR5eCr3dvcF}@I|Y(n4~;(V+h2$jVDS1p zCN!N-K$jS-XQjSOJ-Cue5=gc8SDlisNZO{~Aq)A%+9*Sj834>}_RRzv7M>>edK zK&J{?`MW9qQB5?KC{VML>r97`vtH>PV_WQLYjfL<&Y4!lql}~gDCz@}ZuZyK^yZpRA9bbn&HK^nm4F|0yQ;dZ%pIla~sGDW|B#i&qONxj;MFFmsRK zhgIa@3}MP1=JkfHB^eg|Ax4J4Pvy<8-<^Ng>7v(p6}CI9ZBIy zeb^$HgC|ezP5-LlcabI0>&?4z{&YoM>51@{;>dIXiWMhDr0FLETFiXYe5e(;ExojG z>C@O6>@+a!jNp3)%lt5U(n6hIXCL?*3*rb<|6y~N6ypg!82w!16})ymEEas4Y~vc) zq1V-0@v(YBxr;R@2XjpUVw|ie5My;2l1O_swyeQz87?8!!io_(S}_n|yU_C4AsI;f_2*LZ(HZ5( zHviaw*F|PdJIfk`sMNeiW=oU(m4hz!$%TKKTcdAbNQ+_}t&veb@$RHQXq|pyp`97k zjQyfHH|nXic=2hKX1@)im{$#2?&=J1%ktGZzWDTl=z*fs9!Mt3KN>A%ZIujA_oA+M^AKtBktx_Z_H?`n`Pw_CCD;s2!sqE8 z5|lH(#*&(R)3p7k*~6X85*--@GfeGC$COrpa)bq}=ZDAzCS6bM&%s91F~L|ZXAAJO zAisw4v($PRNtN$MnA*37vv1Lm{I)CJr30JEyN27kSlyV%YfQL~!)zS*#b82Zh4VS1 z2bCn1--7bMWrfo~6n;cjbfd{6#(V06nesg$(oh~|hJcgIL{Z9mNRHx{?}Np_XD(UM z6937w-Yn=8UW^Epa`LB!g z?&JH;@81n@&mGXxy6v^BseGzpOePt(u&rw(VH3-0N=7s7yQEcd9lf|wrJ$0v&Jn8t zR&tjM5!0?aQIfhIdcDu~zllD$GEs6Gnr#GRwJ@ip$qIGOOtK=PZDq+bvK|vMA2|LF zFi#5(xR3jsR_u*J*#3RqsQ9@Z$(U3+e6P zcSq5gWAt`TpODIF5B)=^oU4Mr|*i8G3ndW8)RZH zs<&@8bQMuVQY5-}EI8ZINOaL=s}m!bzOLw`bNbPM-&3nZ(T*?W*Ms~qEH#T9S#@+$ zw$n%L1IrkhI)vD=;p}62+Q>-B7tSW!L>Pe^=0<|HWlT&Y@_nGVe@d$SGOM z%ZuQ1+#JU8Rzwm_-42A6qG1zU+OiYJ19%ekQtY8PoYTET^pjm0C@b=FIN-3}uCIz_ zOXFjVWYf+`B$uWWdev5wbF}3IBTH=LYTnuSuD^(4L0n zEC8FMnoJnGc(p&~cj&0#^FAX({`}DwT{*FDI4JM05BT?PQkOmFOa$TZ&dPR?crTiN zYH0<7TCjjRejh<^-tCi7FdqAyjJuB3bJ#?aR{>u}Ipe)Zx6_1zs|rf9wY7Uw?~R4| z@Zbz{H#}kF?i(qYCRkfrq@CM{qCGrL$zVtqn)0m%x@dcP!@C z+P9(6|M+;eoK{uzVj#4VdgDcjI&C?M2s)i=beRW%_s%!ArlS2@Bd&$z@BIYwr9W3y zn{$wjE54@BX^-qj_+^Dz=(RQ}wF|2(8H`jWKyUuSd2Wf>>In^)b{7|E7RGiBR_`IX zjD>;Rf&Nc?KNepHZQxym42<6*GXh`Z>?C5D=gU<4kCjpAk;@`XYoc!nJrp0h4h@hu zyRLXTfgwnZKlF%IQIZ~2$c{e<``v#{1A4b8jY`q&%)#*mtzfGiS@W4$tKf8U2bZK=vwg=EN5M^RvUQ|OSoy8QH;g`D^ac*=04tp6`M_4USV>Vbo`i|&oruJck z(i){OeC$>eMba?CbRzq;Kz9U1y%Ghc(kGP{i#|X)` zUPJB1h!FJhG^R>QbGX3Zl(%p@M^6mBgDmFbnh+x*0I8C)My2WmZ_NAQX1J-w=<(i& zLZesk3Nu{fVI}qE4F6~A$|M^@w}*6r+#IQW4g9|CqMzwdjYZb>irtNcfA`qnylz`) zpX)re$aNL3g0`O@zi@<}jfG$T&|tW3YdjX;J-3++d?#zh1*OYHX%vG;OfK~zlmlnl z3A^=xcs5znRrZ%LSovyhrQri(!uW(hF&ZLw)>m?A)xs`w5U|)*7B5?j zy0I@)P!efz&e#dZ$84U(p39gGqYptq77H#JoM>WSQJTHJ8Be%U?lPD z{FA9Fi3QAwneIzUHzuq=Ta&((3L_?}X1Nm#QNo^S>+y+fEd3(6FW$0ZSXM)Sym$Pm zbvoUN&ra+4acxaf;aCCMAc1g5G#3`9#Y_^a+^40p^jo#57kb3=UP`CqF*iZ`W)9Vh zOr|k7N{7nOAp1)r30xf##>Fpa)n|(ur@uP12k1h3Xk*+5T6$=YPX&?f)(Jc|Lb)<& zoIZsv8JY6ZjQuA{ey6+tV+o7-x3ABg0I8K^7s1wK^*C+;rJ;U68U>o*aknv?r6MGu z$uSTaP$N8D8cX94so`m%D(l6`6DHsFTKw{7$Lh=C_Lq1FaH5FU_e3k!noKtN=9v7W>{(;V7L(Sv>K~l^_)=J~Kf@O=<3y3>pu%9>8RYjzorc5> zt6NW}V+Sk7e-qP6FtHa`2`!vg=;e0B!NtUipSP^#Hz!}fe_6xnn8P@VoE{=xa1Yp% zyHLyt%W4X}j`Hoo0Gy?n4*(4Qfxv6;>xyf;O{7UUbZkxKeN#nEM#nDWMQqCyP3g9X z@L#d!J>lpj$n89=p0LXhjkJ;QvmG#SK!%R)zb(%nE`X80Ek>Jg1i>=D8aex6d$>66 zEgs7I`;|U#BEU+gl@}+D|8zajHgHUE+=cdDA~y(wM~n@C$j-oW9Eq1gk~f8BF^A+W z_jdt3KtfN34E#b~vxD4vAO#Ue-Qy76naXQJ+5kIo+%dn`r_XwkJD_}utj&I$I)w2S z29JZtmx@1(AEb&{W-1EvtLeAICLW~LxwGdOIk|=X2!4%MYEBO>Ikj^ioSUTOLz6s7xWiF-^hT`>1{ zeq?*iYja42;yTa}f^L`*W|I}ZLI_W`*D18o4*!Lia)n>YVYvPLme(Owd1AqJs=4#L z>P>S(@E?k6SNVnhW4M$ME<4aJpPMkH&M7#JlU zlSq||=p8^t>vB;w-00g7dsi@QDTTapD{Al&%D-y0j~pJ}5O;VXApHfr``)#+pikKK zlj`?5XZ-40#|))H{WdDg z9ym716W(w}l>Q$iA~C^DXnb)E@V_$8n0pMTihq7Q-~Hj`rg%{~A(NqUs$~#{3RSzl zu({cvT9eLEfMdD8VnVJX5G}kr1Ann$*-jVc#{3F6MmXO=;67SAP zUVQF$voE0ha)}gH-pNBS+h>z~nU#B!1O!%CX-fxPQoQsgelPbK!)6^o{f)twuK#_N z7s!c6LWZWqTN%W~SLuzU^(gJG1ZO!VM~G+?)RV=@K%8oxCq$@| zW1*2J0+htJV=sLEWb8kl>CGa^dduIha$|Y7+%ns@tAYw@@lp2TjCA_KR+(HGkoR$2 zQ1?03;&xbZLe=d+_)}5}-cof4AADr{9cXbUZ(zgZKk~#;iY?)*X z9%aaiUL{4+X&ImRlg_~Jf8MAj7hbQQe_)m)_H|u+Ge}JIAwBAmCDgVb-I$ZY-Htf= zT4Mc71zl8M**tg4IB{`iOosX^vzD&qqzI)sc~GqwQ@Yg8@y;(;0;SO zrNlbC>VN*qwG~(GAoFk)n26;1Oh^MHBv9Q%DYqw4oVwN?l4#|=DMxOZl_}iU!#mHP zB{*YifoKFL8sZ-RbD(0j>eyPYA7-{OE?3W|a zz8*8KMl_LYY02Uw3?1+&2JpM1aZ%|p&Rm_V5@oAV`2`w6Epi%NNWh29b(Z~1*840+VeDbPc{UV+!*fG;LS`kq^UnhEd?d6JKa z;-oTZ0LEb|1K1lL_HiRjqpiy>hL0(uWla`wt@9Ga$pg59ZhW8pDxb|>N^kMR9d>+x z85s(A4#HxE6eSoX_-N?+q!Ar+54)Xhn__meYt0)ao-%?b8#3iqc0JEAy(@-E>-pg1 zOFQC;tzDZvHko2KUo)u=*{k=alHD|?=s)jdkKG^vTG(3Z6}|K$)K&oQv#FpK^dSze z!~21C8G(_T&MyuZP%>TaKsJcBL;2Bz@EmAW@+mxd^T3%uFtJjOvGE@4KU*RAjpjuN zV*v=S-7j)<2Mw{ue)0jGNDR3qpQX7>v5yU6TXX3pv>%d&LIAhhA?`R7>~C57AyMH{ z;imTc3*2c<*0)-Xt)48v$MwYhq_Mm?)EDH?P zK#fllzwYnkM6pZ_4+>CDkI~Gy$<&T!2>2qFasnI@y6lz)YiB`R9I&x1QUy73u4?I8 zU)d~K)Mc>%4v0rz^-RDf3=rap~a}Pgc`3IEt~x+hC1TrDN}0cT;6# zXzEH=*J`36k+V=LnqR#?1p~_yttAiVr(B!YytCn!E?g;F@0*l%f}Qn5S$Dz-4k)9u z)bR5y>rz6vC@%Q~-k<%0hIK#V{GX$x{!>=)|DrG+FE`Ksx29;b&_#t^yWPJdhAHHq zmM?2zsxou(eyu3?kh0kHMKkvw`Mu-unwY{6B_C z|BtsVzn3eZBelR5{O2cG&->bv|IT)05t{eE12P|af}IV#&zB_;|Ja^)aAmvq%ju;h z;6L~on6xu3*Ozev2C9mlpX89{W;jALfzi2>k;iKd;wVd)^*ioe$1h zd){|R6?MbNTAv@*zzm1aZf|nC{obY@YA?rXe>E1ev2)_58M2ptGsYU;oB!HWxAQ%v z52hD;jC+BMK&I31q{3A^iC5k;WtRtFS|ajMGSXX&x`)H<`0h@@p$!Xpfy^Y)KBpDJ zKi=hvuEy=LXOQpQSpCbF*cu&9gqF5FmWKxA%o*E|;!E?wkEOEx z?|5{rXVMK}##QeN18A>J97vxnz%mHD7@ki;Zy}+gc!$^=4q%Xp^FC>?^zRG*rRxRr zv5oiukLLDmB(Ts|CO^>hX(^F{9nR5~<|&^Y5`~0!SNsi~=2cNRj+ocvzecJIen*4- zUSmdm%aj7d;p3Ju-oS!7xDoR?HlZ^B9M2g@39w8VkZ4=~mjrm-uHsN4*{@)d)@ITn z!qA9s@;}0{eCf!f(TKs>Tv2U~f!ouw6Rf1M-T?o3tyrM2@IRyFf33~_+)Nvx9Fg<3 z2U1OUtjp2H!3^s&-G=Q$wxFh^1D8U!R-Ee9P8QoD=0dHMIJtud8M+d5nc?6CMHz$% z-|Oy9>+8sVk3*5?ZJrX6H%mv>wg9zy*gezNe4}?9!_Pz%TbblO_#(Aa+U#EK$_H2h zW4*g)c({3vI-naSJ0cu;w=@1B2HX)0>E$Ra9>ejXG0hM@!W?rO}^}Y(E zAHtAQNO+1kjk+&}hhctjmP2THSIlDSO3z8kE)W(Xb zW3M5Uw-yU*^rErJ8j6rk@2i`HwwOD!?#}e!ur)^mtpvr+iaFEj!CJ8l9#;Oro9UWP z;g$DJ(f#e(VuH+i^r&$}L}kFHx9H$v@@KSBsV$f~rLQ$95PYOc`+9uWNl2S~;(f1eag|XUk}0jK8un z+*~+q)+awDD<0K^Nj1m){!H2@#QiJVFo+`DcC#wS3;^oS6PW+vfzQY@k zjT(=&d|JBPa{;2fOWItZi^kHRJ{s~9syMa(98%=vT!9n)+3M1ihwR90vn{HaHITmt#REb7PrKF;OV@PkWbxAnVy3$&89`?Hc*|NMuUpHDHzNoG7 z62JZg>Q#n%^#Rl%I^HB2RgfM&7wdu!M~K?Hwck<+B?O$_UPZB;hesPnuWnJ(4kiz( zCUtYe6`7yb-sef6P%!DT2o13B7saOa<*HIc+q6qJw08(AizS;oVfYS@7wo3@^>i;M zbO=lsJX2pP0VJER!hFgVM0$4#qJga?%MU2@4t9LZrMuX|oywFMQ1|N80>%AN0q51q zm!0lI58fk0Q?r;HTOdC}Huo>IoDYKcY@OifH~V{}>$c$rTiTci&fW|+ z;xIiKieQ%E?b<-PApsFtBMDm=tIHcCDA3Coc`<1%Iop*9I2YemYtxXk@62*0z{Bm^ z%rarQM1#IOa}Z&VS^yjhFHF11d1%DZk~^-}OTGtbw9Uv7NNS0inoVdPwL(3rIkexk z>sPaXf9N5y0A9a(kifh3!fynHV&j`L7tbc(t}vc^t9OJEuuhG#xx^P+PVWSf3Ljt^ zL4IU1w=ND-dM7r)z}7CSNXy5$VqLdM1#{kp?1HpmXIW8CN_lOprourHO{>MWpxxP9 zG%~7girESmoLH_7Xz%a-vc3biFn)w(fl;>Urr~hY?OguqHF#_|tC93O!+t>$KgnN2 zW6Xq+<$##1Q2q$XZOAV9MC~D_$`w{>;y3Rb(ty5=+#}AiOEodAbGqA+2&!NkD=*5l z-YXqlIoa%&oZm%lQ10U8bdauyP;%S8{0}74_up_P1ST5rja4oMLKK$z}sBmMkA(%ejEheu`nHATp!jW)1 z2iro51N2O(Bp50*F5@YuG8%j#v_ZTb1>#e>p!Vtlii|aS*-K8T*q4}NG zhyjcnLNKNaSCe>+Z@WY~;w2Uw)dgjf=jjw4k3?_xuFLlh1e8P%!5c41%C@^7Eod8F zeM^7Emzqw|V=hXZ^8Q-7#<%tZM=>D;+s%V9ReBGqnee-DQdg-~&E75DUlajNIz7=oG%P26=9mnK`g%iYT(A z-|DF7`7&$%ZTX?0Mr~a=C0WpNs%!N3(S&VR+k}y$#FzQGejv&yE}Tq`X@dyUB;_e5 zdZIhMb8m9g$OES9h!SL8<2{hMjVZbKgT-hS6qF9Yiy859xLD9;rzDzeBql1WK4f;r zIu0gYp8Y`#Cthy9MtGUahoY&%ttMEP1Z3G)5ENApPsJeQACa0WxW zE<}uuXDn{XoL)+~_c+RZMbH{wn}zZM{je@~CDve-7D905 zD=jap@sr>~9nEDW%}25H&Uy_7@iNn$yDw@r+CGKbGG3A}+kcs;EG}$i0({dY zVYY3pcWtS2PfrRp=vc{Ph1FdS?EB4p?n{FP$DMm>W#oP)Zt1H7!+&OlsgEMHzCXLQ z7~Xw~39Z4xYxDH8GXE&$!Jv%^3JP>LPr#gPHeM(A@lzP?e$Za)plPvDHn}Rba9xHVsv@$ zU1+%u90-WN?FwYE9Bl_a;nx)0JK}d<$#FWg#+-F;$@SYtr0+2It(p*|bq!hX2EGlv z?aJR6acA`X&T-OjzZid@Esm-dF9P&nc7%DK6r!_TM;_|%ZK1Ov1eB3mxtILxb8K#? zfJRBzA^B$XY>Ezm_D+tQp)%C^`=$ZY2g;B=iQC6a7g@1cD6~R;f5W`4*UUu@WF@@E z;Kf9bcM487IFUA~2d~91{7?#`SAwI3NkI|qKj(Q5$fB+zQ_uLd7AZ}6fZ<4VR|z=N-#&FiTv@0=ckA*>|h%Sw(3%yag*@OfzOJ=r&~`{qEI z{l){_(l)_Qxtqz(br;cGqi{@TDZIFHewG|0wNPC77Q#A>>@F@{Psvs?_wIvnSeN1r zsRNwl!9KT|H>VyKsESF=_ff!0STxjOAN&XQa;0U;pC}uI7!Rr(Qh|v8(b_Isj}V(# zs)daRZI3n^cCtt!fN9YmJnDpYkNRmJYQ^i$uXEqmnzr0kGT(~)IS^+m(_9H^PtBn% zG+Cn`nkv)$sk4Tk@ZIcdjiiiy2OiRYUIPf6glJ>1PXg`IZj;p^s7(KgV1Hq{Tax?f za|@q{f$!sb_=JOVFsZYhnDae2+P_rV136ze91`t0sZ^V-ASX}g1udJ#2Nu^2pzp

wz6v7kMotpYxDL_oMQGbXPpIT2{nDKR!j5+>ixU35+ z9m1?&uC^i@V^3`RyV^_j*(iPvx89#kxGzTh>T0b|N7GcT1@pgiexVq&n(~Pruoc+9 zFpRR9MI69rAsl9!@QDgC-cP#zpxUXwju_~@OHeE&I77UY2$khr#7<7Up1Coq?scTn z$n;uyu5l|N25X&d&Zgkjn? z%uCTbt!$^|#AoM|0X)g}W*IC6-G(>h>D!e1@`^FEvzN!2tF1z(?_>E_1*7igL634zMdYA;%KA4 zr7(Ma=h8-iqk3o8zn`dam2zpn#>w%0C{%-*l4~~8o&j#OE?sT4QaVJLk4{@|g4E?rb^-jwG3JEcSue(#QwJFQHv7v__#D9bR^;L1K9Ccf;V zx7T^i1Fz5UoOO<<3ExGs^SW@<>yLL^M2a#V%l%qB@@dyuB;1j7qv;CIB-QUty1<=j z=EM0oSgN&nA-h0C(nRv;LM28~Dq=bRr|&dY&o-MsH+)qQNiTg049^VE5Kt^hSS!Lb z%*4(xGKktCSKce5J zYSB63JR{}DSE1HF`V9HE#y?`)SJf_+t|PQMl|;;-s8k)Fb!jwgYG%*dbam^VIE=~H z-on`Uu6%1Q_FU1>ieKJf_yv$fL;HBhSf+6fq zVZS+?t6`9w)s=TQD5`0zf39vmm!AR$`q^B$7XKRGQzlX!D#bEiDk+FS1D90%%GRgl zja3YDs6&u z1M#|%(niZBHNwL%Vbh@m|40>S+13SuIxKt=1A85(;cd09Y&eVwM@scw_dX_h^xvBm z-h6A}L=Nxx(k!ry+5p&x=W}M&Q|WDh44nOLryqWlsl&x5m>2P{oO0ZML^zG7J}0oU zFQ=Kl{1E}cI^SB}x2K8}+VA8fRdUPxV;Y}IWgD{Op$L-hjPU;@`^OqNUE33$b88oW zw+|0bh_Bd&#s?(uOD)9WQefc-r-xn3HwtgiO@!~O4NKy7)>IUu_-<9%{X~2#IH#$- zSZMaHxS@U171zkgM#$KANGvUwJH<`)k*@4(HV2Jrj+|>;H41mjqVrO*N67vLP=kY| zh5ge|G)lb3U(OELlkv?gPG`a4!MOD8*v%`@Og?K zxN~15*v)T|@Oy9LJgv8L6zd^P0=(vcTh*RLU$sht14(7ESrXYfRj*n3sCCxQ`cY;K zOA!^k)Md12TN3&CS~&DM5>-Bh5p&~OM)d9v52+p@J&(C9YumJYq}j^%vI_q&<20g55_Y?0c`if+a}#a(rcN z5OlOb)#Igd?XU%9x*+0?$b@w-XVgc`3Q39ofQ5l<4}=oIn(#$L8TENYBcD_8{>Osv z2`|W18AMjFRon5HBJ-jg!rndhZGc|5S=TTYgaCa1gb49#TI4cL5UfS~uzEm;xuF`X z#hO6BpBn*%Eo1q_s$EbhTGFM4-2ZE7D9M!gUWnCpvqc z6NY2h(GF~U)r3*a{@d)#Fofwo>o7*eDEzY|(A52ZWttXAr6J?{k41QKMgY`&NSf)G z+n~?hd&dG0H3}JgQJ>UdQ(9OL@=S|xGN!uF!!>j;797By&3p+-Y-MMS?Q|saM3%z$ z)YcY#7QXk~=J0daZ_gbo(P(h5V-5HhlXAp!Bs+rUI9$?bB827J)6By> z&UOnK=^sew&1w2CICKg4(VvPX7=r7)u%=F(_w1Sl$pXGMy??2%M+Ou+1Z2TIc>~$A zZU-QN0E5q|(PK4Z&X$}cOv{9HkN>vAE+ZBEvZz^4i(9KDS8C<9H~1GQtx_sbx-QflYIE^x zc|NrLDb6`GW`B(qwLBrgR|D+B&C4}Gy>>jLkfg|OMspjG;+Dk7i_Ya8Cb-#@GxJ+V zSocO;qW&Po>uagH?{icF^~PVk!6w+c?KAzX%(+V*o!!^@xAqI3oz~!?@-#Q!nR^yp zuiW%aY^i^^h<$~A(ILHMTi_Q)#ug>Tz0Z*zHS+r!?R&zlH1qTK5aT|j^hQ`<(}ZyS z_6tX6{kA`Krg0zoiqCQYa1PVqa(x3PqmH(s=9XlPMj*KKUH4yD6blGK%jtmQOX>?v zJWOlaPUeBM7`Nz@yOFX8H~t9t#1BqyAJJ&n7b0s#e~jz^8#O7R$8 z;;zipDfNgg+{ocHR{>8td-?G569GE^|tWCiooAl?!Y7LKK;`&X9Xd1fTh@GG@xyOJwroVrM zCh#e}cKCkrs6UP{E}Qj6M<2a}N1$~5zGle)Rr1;~tw7O5JB?!12=uO=f}VL@h~5xa znsptTO=6fG$_3FS5IQBGsR4`TQdU98{N~DMs1SF zxLH!cjm6^4`owc>M3rJ#;t)}4=-%EF2(NI}sV96z2vzJhNB$3}6TX^!bUIumj(oRQ zB8VLjN{7R97=#(=fx<-t2ReHa+U#ABfOgkF7KnH*aPV#pv-a}f>$5*`)_eSZV+7pko#9PKt7L;ni_mdtB zp91d)8Sb-8#d!Y*%M@zb1y8zLXxgAIU$OhBJ(_CEl2F^CsM*mhg3byfj>hUSKZm#v zyf4Kh3i6|$ed-FQf{%NIi4}t$MuYi{k7b%YlRau@B@Y4Kd3v7t6p!pqiU(sX&t!x- zSmxJlOh`8>S67VL{OO00niK=SgK(|;e7+(#_x4=%quuz-g&U|J3!ew^z1D>|EQHgr zZQU6A(`K!Olk~q-Vap8^TI7#xDmN;J_;Vpk8vSMZk|8%KhKbLGT$~%75lTsJjVoJ) z?PPkzl*~RqA$C$~W1f$ao)L;gm*7#Rgd4T3(x^NnJHeC;$=RqJ=3xPDnQc%GF#}(( z#MP+0i-Psgo(Wb>Ijv;3c34Gqe;6WO72TsvU&|Kh*})S=J5NeP`}4NM8; zIL`QgxQpfJ&_fk~hiPqrE35DChI=Xv_qVRA{@*V6?(*QP_Rj{P$G7a$$Bc;WNK4k6 z5)V?zJMg@2tKi>s=A&}P_Fpsoz4MEmyW>VqVR&v;cm^9nb+1DZ7J}&5=#X=wq#Cym zIMQ+7(Iq*Vo++~839r@pb#cd*%Q=xH-G|tb%|{$9^66tzLpf-{3PCC~&yO($EB8x$ zYG{E+6I1fwTTefj3YPa~`;_4gE+q$O=cAmTXCyXTFXt-$l* z&J_m<6x%;Dg!gc)ze{rZ;ffO#o^1kHVhm2aQodSLPkkTb($&`im=>-($(;h0*a)Yz zO@`1ljgzDMJL?zgNEjghbyEnmY)dn*d=E7qRZMTBsk|gH8NGG&FdhAr;clf72)Q#> zbE}`~k&47c776NcPG^cEPcyRH`(EiJ@7+eYMu=Z9^oUU7Woh?fe0?eXg5FOrMU}k6 z*K~E@f+#*g$DU?;qc^DEK&MIYQB&1!H2OB2O;7q+!bMx|hbH6~y1FfD#}>u;_}qLv z>IV?7U!;Tv29~qW?(uYm3Dnr5+Gc6g4cAI44nBL9(eh2kUo~r2@x;7XstHI>WTFTd zZd5$yzhO;~;2MX&kVs7IuN_yJuR||$otdyvqQ$>LnVt_xNmgb|)AU&EN#n7Ow!fe} zS^l}?mh7GR$Rkl*Xo*(pPVK*bdmdfF!f*bmv&(+l3^uIvP|YGf@5AzaXJZ*bbm=Cq z)U1+XkzwiN!P}3AaRDZzGe4hSl#N?##6`2;2p!q@OlMpoIXx`Qn)EFfFAv_d2)C<` zG#}>R$NrJKIkO48P<4{*&3xMRv|*87*nI+lRWRA844b1?Is7>(G$zm^#!hq8HLB@c zh!aR*(MBNs6$*)Jt{t4)f$b-x8^!G&cZIGKMSDFiii2683NaiN6*xyu4UZd~n;3(z zfLsv%NxSOgqU^m4x$5Q>yvakXF~TXt>2)tQO`lC${<^{_zK zJ9IO#+HCXmKd2`ibk+aaS%jVK|IS;4hmVT`Nm!WN<)^c`u^p1f?`ngKG4l?)PZ6z~ z&2O6TACScfE?%@fdT94hve*buLU&T&7a#T-T^AnjI>ky8ZnJMe=d`7BBJ%0CM|( ze5R2-E*8-~I{|<_|2~FA2=+ejr@^4zXC>DQ(xL-^9gC2tCRwyAlom#8?>(ifl88*s zN^qvA;VFe^5?nrXZp}!`d5_2W`Dr9&GEPMxy$Ay}0uO5n3C+j>Ij$8l`{OpB#~j~d zZeO2t{qzpB7&0L<|0zC2N@LHwxglb!&>O{_ZBD)}t|%R9MKp&=^Hj1Zx;Ml{sjwgZ zTPSKv7me^jAqXYe`&v3wn`lObrnO`f(s_??w&WxHw=XuC9vWd{Nh3;f2iO0xKcr*& zr3}MPoWEdy^Dn>EdYe>H6FfyI1Jf9@XcFA{s=#i6f$3n>4bc8EI0 zm*u9WrZXYLGKVF>Dfu@Jg71i83+>nS%R`#xQ1H5DLMkOJc`||JmEqm=v2*~)*9zmb z(&NvKus^?);927uC*5Ki;?v#+*aXP52z*u+oGABhB^ zu4!n-peI9z>d|Hu)^5S=uT7@Cc9mW%iT~ta(a?B;@h!GSFR7d9g?KF*|9zUGpuEJ( z`)6E*a8)a?TMQ|jOZW%lgcieEQC4npi9TDT;`~>9cfO;i7fP={8dXAQNM zZZR4Fgu6nVPS*VP91a2*Kd=v8+l02_7UsC}Pr)h91uUCR_7lcl%EVIQ-||J_3BeDfM!1S+OU#32 zi!G@3SnowJJ5u;bVNh(2Cfpa)1Kmu|rB)5VA08uU2BC0%bf|;D-TfgJ9S@RD=HTL_ zuJZ!FTy3d#$FjpnI0LfzkL5{vP#Q(8(kBRBPv44|IG^d@O{>vd0@+cz?E$O_nbm7e zK6oHE?02RJ-w@?*=i)AK^#vzP2$ZIE8k;%@tkosO^Qy zx!DAcwc_gQeVru^gzzf4cl7hT&g7!N;>OZ7D+)T*C$$g$qH7gv7_+a5FZ+hCxvz@g z9`v>lUeKx*jAzBhl3_6*@{J^_Qf>Q_Qk5{Fy`*3)sSlvaOYklR?d7@K6G3*rt>%MP z@k1SBg%ocbE<^4{5}OV`AT_`numIU_SR(E*%M6c9>5jmfX$^rrZrEH0fqUqNwTNBy zklr=V9S|@iZk2v$B*>CpMZgGUv5E&>hLCO-13xLriq{|@zs{p9^U4MT*~>%C{YB{8 zv8o>L&CE5nzXXNnkqhrlYr91XddTo>du&LKKE>F@SDQIy0Q6cc=j559lNF_DLHKC+ z^ux1aV&XIdyv)VN>#?F0)tdZGdX23nSY8h{6sR$OLiYB&QHGAiQte9Wir&-8y`H2 z^>6p+a%IJZw@AQayw7%(QsvjITlJ707u4AQ_4IZ`%2@{W=Ab|~KAFb=g zkNKYGz4^f32X`n>AJbRZLK-FC^&fzHmwp*vGl8$k6kaDMl3Aj($taqEQz#X(bYH(= zwG&>@Pm#vz7Df+}XP{s-c-U#dqxOV!2JLJUPaMxGoCGIgoH8%=;%mYVKA=83XPf|SAmt1D`E6GArrQeN;vV=QLL++kO z&-PZORq@wHSQ6L3*)bMGYk5fq(}&JngLnm>sYnBJ{3zt>E7(C$U5xq0;Z`1oc(My6 zUuj{iG+j;Dv%U^CM4mGi_uYw~aot6FC=xvlPE85k{BHR*sQ5y@dSaQft4_VgqGT0j znuz{TC4>Uv+5TDn>`x~7w~lwJrJfo{hbsP8>GmOC>kyOj=KU1NvU_N7*I&`pU$2P(#6|@L|Fa za&F>bVi%MjB^=_w8qdSgaIxW_4DeHXVu*DW-B4=Es*UL7ADeZXrZe&IENKszO!t} za2=USI(m5#`_?tqS}f2Ub%uAan2j3AVFh2W8JfQ=z))FKb~cCaV7mXpR~moeLUQ8v zX@g?&qC>fZp+H6zYViIxQe=Fji6N^aE8d5~G`n1(@kO{!0=3v!N0f(A7niUCFJ!kp zbQpQEl4*VOf=5&(w>sXWfFZbWK{+7un#m}O#p@pabF>xOJQVI-jnVb&OrvDI`1$af zoQ)84$EOdVxgI!~Qe|IplPXWQsBE+`#A}7n_Ycwi6+TdBnG?s_u}%-Y|9Y!xSW;`2a-4Zf2|t z4I`}*OabR-Z&a0k1Zp!_AfQSMrqAELsn^NYkG$^Ijl@>~be;mFi%VK*Ntjr)SmMUlnEOo3Z;+Lvs)wta`-NN1#+B(ZK&|;gX*&BM_Y*h>&OhaPlo?G zN{=d#(XZN&)SA4b!R~M=y|wNh_f%|hII5GuzfwAtMO>@198fWsH259#wRu8gtyhT< zZR{LE7Kv;**eGjtKfe<9t{$W0!pCoxtEtUg;r431S2oS&ZT3+7}0sXDO_!ZK#!22sG>kH?eMK4CMlx3ID_FW`&vskRgD~9y_hMKry89>SgjfG(LR@c#k#zJccj) z!$&fQE|yiGBx49@hCBa!+c^#BW}24Y8^4_ZbXUEBfkQ!XbA!j+b3>%i51-;vFuDUK zXKbz2XVHzhT=oy26)y|I+#f7yREzFHJOAKyEqgztsZWd=cmMA2{mXBa?qgwB<6>BT zM4k0SOMguVpeRCE)xZs_??5lA3te^~lSQ?VbKI7#j)iBdmQ*&;+wOoaX$Ho385)L4 zi;%?*%T7ZpMF#C)!abw+E+)89wjf_$5?r;)#?>067Q%STw@|YI=j6*?2!uC)pZUi1 z>V$7QMaJwH0ftIeWsUvN_N#dD#dEyb2K)JHIQ`(G6+Ib3C-~cBftk-r^sBm*WXLj2u(1a=`K0NHn!!6*cn5|^mm}Ey<&;HI@&H-YXwFQ-IlJ{HkFoC=y zF*&0zjXO@#&x}59YqZnfUdw+VU}0QQ`W!qe-(b-Fs0MwRT^!G(3bYQfdmSJ?H;$qM z^^T+G*tRqKriV8U8P1*ayy-O?QLAj|L&AU$a2b^=LkR{CXgE-~rLUF}H3a>~vFIJA z7vx4OOL!fp*?FvDpjxjtxzgc zX}Ntaf>)FsvxJZtd@-c7%Mp(q; z=^`}}+HnA<(0&ck!9J{@zY4~lMynAW{p9P;q{3B8H`59f_Tp;0uT80DdhbJsgg)G? zP+yDs#3cu1pbjFCfdUYN-j>sJ^-KOYf>e5=wGrY>1dZ)6J>5)&*+L&(r^CrTDj&{F z)XYpXtTq}qWh*ZT>JOJnBF?|N_1F(jMHz}e82vE?4VUsRY3m}o?Pg~4QYsQW1y zJRBuDITW_*WO?BcWSO7O9s+x29q!WL%6H^7TOqiI<~Cs1R1tywjsW^1jG?P#^UH#d zWoHkv#ya>MD@16j%xy=Nz<9aHSjp%>5b5iFhU0eqYJMR{&~}|zhf7tNA^TqX_#vxR zIR~YgX%%f85B?3GD}r7~O5_$rFEl4g+|2cy^YK?+*HACESXnD=vydy%A|xs783Um^HTC26%?Dp*_xC37Z9i2P6Cs!Z;2+D*_tN$#hGf;uvy> zeBGo!GQolA!&k@&B#V0Zq*sL{-rw-Mh>VM17x4`-xl3QmKJODKRZwxx3YqeE zc2bNh8u9uuBwOoL68SbL@+mockAvc2t!@KeAUxK2bTxU=IHVV9w}bV|F?#x~Zauob zW5PdD%RMW%MFtX{MW?EW=D|fYS=}1Bwkvhm2y3KSHon!VdGaCDDJ|u93c_uK8}n(l z41|-6jVr8wU~`c%4F_eLIUg-U3L06uE0tc8zugvM2#&TN-+MtN~R zW4E<>>$iw97A^Of2S(`3iyz^0g=}<`&ZaC`Y*0A>lnr9;>&SBadci7*zolvwzyo027 zk0JF$K2yPPt#UlV)KZQ;jvS6DnwW8qas0Vtn{q1n=6gx1Kbg{K);r?RSTVcaMw~A* z)f_!Fd8Dc0R|^p3hC}2Ot+6a+eNAx>|RXVJ~6n9QV$F zECkf(LJTH%BbF(xaq|~nR9(V{Loa9_ohNI2GTcaRYTJfGY%%fvS6zFjFU7{wWa2f4n}%)ZpqUh?j*;9S!i zCsgLioO${jCqEOA1J;`yf|vxE+#h8K%wRS`(EVGBdo2<*pyg=eGXXjeW)}w+-SlM> z5SPYO2pu;w-Nc7K8;}yPy2Ue3H}O#IuPJHrjUnL;i|)U3{l#rk5eBSS^2RZ?sh(x|v&cxDJJ$W&syFpk~uv09Hl#8#VmHkyz! zkSd|}@O85kufRcPUmJx!W7kV4nKAj$?)X+WGxG)8VsN$i>ATBBl9 zyG)XtIl%L7eZ8N1Uux|!my{nO!&}bAH-(=DNA$xoEF0weukuCZ?g0vPDOPKb z^WyLdkhXls5}7az2=3VNjP=}kg^lf84`_8KMOb!geqP6U3<1pW*)mDwgF_^Bb2>D0 zi^pzU-?80hvJ@FF?Dk8;(H!%eQ-c-tlIUS~F7H%m7Kg5ME)}|h-S}?v{(mH3chF`B zztb#QK&iRdoRS#Lf2Ub(`M94ERfEt&<4>d&jmj3ljm}Xou#=JgyPl;BUk#Sx$C^J_ ziVL)5=b0Z4zlxqOyL6UAR@o1wYp7bh8)3-}wicE(VBV4BJEGWrSnU$W8@xRPgoYUm z4=LrhRkOO?AA{)6aseo+&m4vuKr@T0V{!$ZapHU&_igbOWm6;&7-(Z1hT12Qb}3A z`)?oeX3W_zGV0e2mOWp|Vyb|>4_`@7X44IbZ1_`uC?+;*-gkCBf4mz7!pt(I6PDd9 z!2K~o|94ZpUpn`$+F!nrz$d{w6k+}Yz&B!WKK`&C^lg)A z_m<8ao=~vZUB1sUC(FrL2s*RU(CLM(dq#$!mC1#U!U{^gy^+~c$;5qcW#&!dw@{_( zk2{J6-sf(%GJj_~WCcRls9lrc!YY+K`ZyyfWN3IzKxBPOz%xqf*pu73NlGjmEZehR zC-q(5@P5<^kGq#O965z5VFmwl>FPX6V=eHZi@G>T&*y`ilEuuY?4@v~_Rl-ks{$dx zD>LBSqKxR$bFL1X?)}-WGByWrUPMnG=S+m?%>hfUR~c*g$3&^5?FN-Pboy?xDHOc} z-%BK;;X1!UWJ}!C_hd^vYo9eASZ#T$HRGT?e>9V83mNgrUzWg}N72#pY>wRfBxK^9 zppyd*D|oiev4!f+mJ2Jp{c-fG+=_O4t=+6S6LEV(Qv8P%Fh4H*J#}G(^s3A>?bq;3 zsbJ24&Z|lLx(Y>4LLKW$*m$cdyJWSAH2Wvz6mrke_*%&-+kM^N_#BF}){L8-EA*h~A5TiS!c|O* zDEMVDoD(8H&AEby%sxFkMPY%?#i%HU&w4SHm}M zb+

vQ0ml|AZ@^j^#j9@p%p|e+Qzip_^XQ`nY8SlyLBUD$bjtfqm~m(xw4P@%q&S zK|r&h4W*W~fNWs^vS*I_(10h-apHcqvi@#s0{}grhBcx@vUW zZLV&~@toeEde>X$S#moEPQ&VYqM=x{@_8{Sbfxa1{g-48C^#S)U8JdVi?VhXSw12F zJLO0cZS~L&e>0%K?o=c~;vnsA%pg{Ou(Yl&C{gib{OGR2U+V$QF7q%~lDnXqvVnad zXT=5S((ju={>>BgX+!lc9fOx=Z-01xE^iJv31^ z55HT&3y}VGhodLzf6(<7P;mx9f^cwm_rcxW-Q8UVC%C&i1b25QVUPg9-GheUPH-9A z$xn7~ckkZ)_ujnfuI{dy>aLpaebd!l^(^t-nQ#`Id){-f$1|5JvddQ&e9lKkIP9t- z(qb~`yaDFbx^(7L%3WYS1X^$u>0LhMa^22*PVD6&BB2Ok#C(+TLiPI_>($=1%4qDh zzyKfp*Eg{f2}#M%X(OF6@j_fB^Coh`8t+!M%Z#-^Vk(wEb&+kXRq@?{r~@mZ)a~QV zfhe!YqCVS<*SN7yew;fWHw-?G>;zk^9orvi9{vN`wGbMJbIZz|)SYPIQmQHqT+|1*WCF#JT>; zr{~Hx22eD^g`WB3X#q9au9@WjXtEJe@x@?xvgn*5DOSI@JGz<9 z?P8sEx|alTuhLH71X_EjrZYY(b%9RkP?S4>+fueZD4xV~O+rHK>vPxTCH?+x-YraM zXa@z?{X{J6-nK=4YCe|uBE2u9JOMB-Z8J$mPl zxx93y?>^mw;~Lr@tV#iLee05lLQ051)+MAjZJ7n}`kGA}2n1J@09hh7@Sz>K&Twge zKSp%TyJQm>3ItOF|44l=n)@~=nPOV|R_~1yP0?%gjI_asGD$yNm^Z&q|L7a~xeaLT zydDvB<#8TA!2bHqWP)$}q;?c-sy-NBIHAQy^nzM7_9f{L_P`LdQO zX4I2-P+X+&SQ1)N-0P-RH)U_-6K&|$=NnA0ro$dYP~G)MCG#5U^|9JzyGm&U>b0fE zX}f9;dAo|K0q)Vx;RCt)5n5FFjPjaJ{rIUjLk2QP!4wUw`Ro_6r9cTGI&3Aa@ zQX>Y71WF>KLW|I!k**1g1T&f@C@VB{L^K$r(+*WC>bbV$Dwy`S;CL)9L}MHd9>>OL zlB5vKSPWtbz>u#rl#6C zSCsC0ZH?DJ($;*-cwW+jbSn0H#RIkA#gR!6af}r z5Wj~}krKkle;oUg*E$Xih53LK?_pGyB3SZDzLr$#!-5x&86!c&ln)v9!M%!UKO%w# z%_@8&c}^;KiEPUV#^Cx#h$RRdw2tM&U>s(5d?9!?HM+x5ibR9Ha4f-WY+k1R(q>1* zx=8(%5w(j^bfLLmXA$kT;Dw;jsQp?Pbvatl%sNpO7|~QYc*LS+o<8<1VEN@E1Qbuf zHCsD-97ju!**S=NY+L5QKDnC}`y-VVlsKO={N(X}TOf=Ifr1ThJ{)VHw^zzSv z8`L!ue4^Af2BxGl(XZoeOfHnSXkCmJ_!!^3mMK^cI;lLZ2B+mt0)JD)Bg=7l$Wf@g z(8(Lq+a|cizU7^+e$ru@T*m_l(zC_L!~b+nuVfQ>4J2PuG|ZTjH^V%h$yUz$)YH1poqALzeEHbhnlpu5$-9ssMjn;p=BS;E=DQdtY}BW;3(v!$Jr4S-eN(#pxh&cl~k z4v|&L%H6`s$r9+~@sGwGV93t?@iiL8h^$)9+D>*KBvt@+j(<%bzWRSM{fh(fA7lPk zK#2cclz%M#F?6$Y@o;tnu>X@-bf{6i`$dPG)LH%lwGe^aFUuPJ^sYI?f3I9NG; zOc}uTZw{=w07Eu_Aul(ZF@T*HV93GC%fiRWVGQ8p`Do?fU}5LsGX`+60}OfC**<#s z`2f8CHH7 z4qfW%6KlE;dDuiUAWCsBkkx0qw77|(uDZT`7Y2A>xN)O|D&_qUxV0tPi^lm|b8$m^ zV^Mo?af4G58uAka2pM|J*1?TOgt$?N*sPkUPtEYM+P&xani$454kD(&=&-$2oebTL zFeF=D;hST%M_7;Z>}Nwy`QpoYl#kj3sb!=z_XRZigmQO0pnk12EBV=lB4(PWOaV? z6CPgV`!I?<0z*KCA;&6PPhO6n^aTa^9%PlvVV6%!X51|Zf}T?op=_pa^Et6U9{?I~ zLtS74YDfKO`hB4C`S)_n>9kJ#P3O&Y!e@fTS+v5Sv5}KepFY*bMj5o=Xwy@8q(d5( zDEeR#4qL-Z(`k%Kx?cd^)@o(vn;zdWJ}O!%Wd*-!{Yun_|( zBq#>7;dL3*zGPX@h+?sf-fNX#CR_V;pL{PK5fFPO{w=%x*v46aLA(3- z)4chCYJPnA0ouVd)(3X?IIhQ9C3xEM(P)KOmy__k8T!;&GCK|gyTj0emNp~CkCeGY zK@#F;hjEG#CKbZBKVFd)LgY%AC~A_!3-;X;@;i)pb}wK-o~nF$gYY!J5eNnI4D<(e z6rkC|iE(1X5B~VOoDYRDGTR0uUL77hpeA{EpZY|3!!~Wt8b+nl&xYdDR$AyguhLF5 znFIfWI5#wgs6?DEL1YpA^s(2mZ$8Z0XI*ytV(@>S#%0{~X8&d)@)rAIZ~BR1$j}y;sGVSGE;Q!zr9Z zdC5*(l?{<-w-e(2@PuQa9t3eO{m3UG`p5>^IwP4U5(y?AxKV-=1@mc!JbcH2?pS<* z+GUHNi)h1evbc&;3!RX!5=!HG$_=)Lae957Zsq%S8u)-;Oz+-#Q+~JAc z(0lpPJSx)V+d}sMx!$OG=5QVvXD3L{;xy$e0!-Fzo_%eTjD~s;dXncCY0>+&QarK{ z|Hhe|+BG(n-cb2nSvGkeBpUg6jIG3*wFFN-vMclh5O#L`aTNK{~^|yPzS~iL)GR*x9JwRwxLWTt7|X;`bD-@d<&M+RkRwEMOJbVi`@`NOV2q z0xBzT&=ZyPq@U?L1^v|>=LO-ud8P>v??z%}6S*=yLtLQ2*9Q+_y$sKVWDAOBzAlqB z99rOVGF?(+qRZ}ftc8Q{b|_)q6Tt){7Q&{z?YyAZsRy{JIe7cPs}tcBqN0XDnW47F zy_+sRWf4}9#lz7*Pub_pOuaYVFiZ3&jlV^MGs*N5E%;Oa@V7oZJ5EsPtqt>+WBoE@J2HUB8_VM6(0>UE#(4Ezi|}SDc;Gc>`o|Z4!2%)@#VD( zVjj4^1-)eN;nF~ZaI=tvUNK@cFuDq%`}u7Bf?}(ocl7QGWn>vykD8#n_xdq-;1p+i zoA4QS2@fjFaR;3~5b9yaz+m`%Z_s}E8KWoE1(%Yq_xM%lgXpnrwtN+qx2}Wz0(Hm4 z1^(?GW!g&Qi_|lZYu*FXHLf{X-^j=_{cFI$J$>f`z>xLCiLLIOPvn}7e*k4-5(k3bu_glxgri|G-n$Ancn{cn)QPe8=)auq z?zP`vga_=f&&DqL1U(XFE_mqCD-E;@#`;Hvxar|)0zodAagnBv@68(X;;4u5eSxox zbQqIrJw&xbsMVNVa(NiPErd$*f59}DD+t*bhriK@$rgXXRnW3eR(TNVe|*I6^`XR% zi6>8+yo35yN0vQ}cDLXwym}A&1+J2H{_W1H^g*PDK$lULIn2H~B+AJu@A!3@yLY4FpA~zB;~zCU}-SzXRT(VACs<^d>?o zcXQZqf+qidvW#6aC!H4ar+`N)i3nwYC;UZdTU-Pbo|He#xax7wTK*LNULf)pbqr!` zo98yP`!Dg^yBP5mS>Ifco;CiGF5z0y-!_5wwvcz@IJT^{UXrM5{Aa}T23Y15xTRzZ z{9vLTL|BC%VQV)KMh|gCpdg$E(V1X;M#_j9Gtpks<3Gr+e{SBMFDI5S@87qRphT-V zub9FBL1C}(hgq(oAKtFGVLb*`)0sZg3zCNiGXcwzf{&a8NsTuX3{Wi`#R+;nM0)v4 z>_^P1qwb3yrU4vcYaRH-D4`I)e_DmKLcxx8hKY-IW^#1yW-H;ozF&<#J-q|{Vp++z z@O-QbQPmIkHXn7aheNW9cH$R(mL@Lmp8nGIeZPSnsDp2K4Hm|o z$ z+C2TvjLUCR)`zIX_)C<1hPxG2CQ@M2ArIr*O2DXCXE{y!f}BFRwc! zqe*i`y$mjdJA$s?ABYR-7hj1JZ1jT;tPf>jZf@?$g9@o8Y~qNUn>hu0oBjcc&6r1- z<4-3G$hshZkp(14Q#C*oEBvg=Fs4R_2Ksl(Wt)~W3pM=Mmt_q%#vcVja_jqaSMwat z41onfOz7h1EfQTIDO)TjTvF_JhLhp(-NjvpE7U8NtDq}jqvkqe5M5!imm;fzsbZD3 zpLWYcDdE@L+T4a(O=dM_^^Nil!@HOh*F~>I!&=E&Q@9PjBVB$y)GWAhSGQ6jt?A_G z$+AC2zV$``-Z8~Lviqa^wDqanQ-r{TW*UQNgDS&@#&2y-v1P8c?c43M?cnxmKE?Iq z_Vo72U1Ha1A42Dtw$(lvN58y&>y{ix_Eu=aTIR`2`?~MU&0j%Zn>HLe2k(o|5e%ap zlN^iAsn_yap<2-#|H@4%x6mdk*2~o^)vM(H&A82Ys++4issY*@>pz{vH2<&}E&M5J z=;C#~($u_EWbE$srtdtH@9ul}{^$7W-lzKv%f{rHu5 z#yJdF2$-|zlQfuk(U>3D44h6sO)58=SQ#(b|3GgsvRnrYyWzNMue)7e>U}F;ayxZ6 zZ~fYx=?}&1jnB;6vE(*AmH;Il1xT#xryFQ~4_@|0o|=$|sw4 zR9p$>Uwt@it%rMsWx{a&n`ivLZ=7ZzMpoQaXEuaYZ-rtm1^qJb>yPv|_f7~|ZImi9+7{E0lZ_DOpKEewHsO1C)D-`(*NbnBGHvmTFqwzjCU3 zi3*3)Hk(g=p1dp)Ek4v-YHyYG7`}Bp%io2cD7Oa}s3m0lRsd))i`7w*j^{DO514;&5UPZ=Hwz0o!j0gO}Yyn6SHA6cs1cw6Ek<7v6^7@YS%lfqD?Uh)aF2k8tbIZGD98WTeru;F#Gc>o=rzh^lS>sd?{y z1|NvT0dMdphe)z*Qx`(gFxqWyADF@jel>*CV#Li)v|We|gQh#mK1iEUtR;k-p-?X9 zg<(Iys_iJ7(cU}4Uly^zDD_dk+hI>?(Q}4q=d+)XiZODF2@FbLEc`G6V3y21SffPy z@9+<2`ocljnSC7X5Is@OK;9Lx3yO4jDJsnTNK!Z6AJOZI7vb*P50$6YVQs;~q@)ruw9c zIZ&k#1K`F5_l>VfeaY^VsW`Hc$;ZJk#I(`dP>ec{)O63tJ`0^SAV>Z;7LRQ7?UabaOF^Ez!F%i zMxlgZ4bK_L7^wlwWg`iqGM#t2=SNzHqIIPv&JD5j)fLxd4bf? zsWS}97GaK{!oyI=OMoL__%wbF1_2tcWa-wCVcKQPG*~ow6?tW%+(LFHcORYmnEaG7 znWXVXcwKo0Ki}LbX0mhwpG(+j+cim?iE^Tg?8zC1cwf4co<)8Tcw(&_xSdB#)p z??60j_v=JmL-+ego^$ux5)|A{f%lgG-&wd|!PgyG*~FmN@hZHT#Q)dl@5;)YQ`=!e zHZBZ5Zf0((kfn=8DKojqG=m}R@Q9SGbgnYS&Io80UD8FZSVXL9 zv06KoP7Sgq@=6+~Nw9H%nP$Y)OP`Q&R1h>U*HAjZY@t0TM%rf7zp@{i@3mhqlC*HK z*Ye^y!>+~+*mV}v2{5^!Xs%4B=xzn{D0vm_-%6=8?0Ep@br=!HA`fyQSCkp{ zCA{Yba$5)DAkUm+L!1I$eX3ZPalDQolaa+H9OInV#g!}&Xsc^nZc!0)#k3|!a7@*X z62GvT5w^n^mXu8xb5yi|0IQ#w6%{?xt}?FN*U?3%N%9Bu&4kECz7k&Q+yE`{@=)qi zyvx1XHrA~|_;Ki;Yaba#7b~3l)kj9)<@;Tc)+0CkQ^hev!>!4 z9T}~>AJXUFzv|pGF%F8TWv<3Ctj4(<<;}$mFIW;MZ0hNp(D3dibL1i9WjF^-KT7Cm zva5R8?~M*Gwfw9TTaxfRueEn%0|OsxHn0s%LD!YF@&CEB5! zZ5?r}!y^MF-Jfp~N$u;aS2&JQbM=Meozz667;8a8zf3TKuh&AkQ7dGuIA7tvSh~&= zeO|u!hyzW3v2Te>adLH}`OB4w)uyU~<*bx)mk5P}f{QG@*pbAlY+>$IhtVCJ%OW_{ zALhT3u3@Sl#Xm=4rm9NszJU09{@fX%sNW#-$UIvcbFbc z^P<^>Bu6aeH?)^ccFzBu*gu~0BmnF(J7DUU09ca$hCfMEito||l~sBHwX1kSj@xPn z(F=c&p`m?dqgdkD=g<#(KoXFXHRH_LtU;MF%4!%@ZX^k%&=vM-j1@hX2!!*B*uz0& zM-O$piPVFaHKIa3fp7D;#JSjZXlpic>1rg&-9h$*g|+fJw2y3DJ{i#r=7l73G`86t zQyNpnhLwbclrCeD{!IUZ(LEcbMbuG)V!ROb%}$beRV?!GY;n1X(04xav{?OPG0=I) zna#6yNNoEz^5yXJe;CGKPuU_2nzWGguU4x#qAyIXV-Sw4OcgvAPj$UXa4N|ZR|hCo zjpwg`bma~-<KS5IH-qgcEed_Qwm%+Oc?Izvc7d$qPcxOD^?b4aDG3+ooI{ zZYd_4-$;&rWXr5WzmyRozA8jLwa;W(U5thI>}{ZA3scVQSx!m)hK`#jkzI;I!9~Hj zC1d(=w*=YiigyShrl@KIJ^GP<$a-j_1*U->_nGTw6nl&}!y|P^(lA$G)J0fIjG$mx zHsS>;x8$0j7P9anM307ihZR|1oKrZRWndE}JEDD?2Mvv0*n@d+gouKnfH`x?lfW~8 zp1LZfh6I%px#YGak@)v`M`B=;6|b@E(#lVMuRP(OJ^EuHt8yy`T+6(Cn!~ZYKLJqP zFjQQmEP5TVMf?EGFQm|d0|Ff#l2fjSMT7i}zUq2WGhQm%+s!GOA*zzjr}Qd+=7@kC zki#OG048a@C>N-h$1c&6L{ z*D98RQTPY4s>Nav)BgdM{@btr3iU8dYPx^Gs1%x7LG2%CmqJiTFP1{c0+mIOP@mBO z4>7&c|Fdp1kod3e9*dklCiV|(V-nD%g#H8l)EsIQP}Cgl5nnJl6vpVt-4O_=Ez^_L zB@X@zhf=^-8wkOKWdVvI{71cy8hKsxE4 z_!Dc{m^e0n3ifkUceqZ-almG1ApHObG|gH<(+1r?VAPryt;-0R z=2;d&M7>V8bAaiT?(!dXqk)8fb@y1WbX(bf;1KhiZZ7d3n5JI!-EO5|i~TD?_t$=D zmPoD!mRGmq3-XQ+eZb@uCg0T69VmQHbkI8SFZ3jT52_oV33M#6MKEz&h_s{t6Mz2` zMm8i3<&=!A*m4AjWQQM`y9yN$5Vu{ia?i)EC#>Hs-);PsZ?O7Jq_VQVcDz=(_Pq8` zYu;;-9Vg?a%nmS9mTj=D^KKM3bBXSP)o(*ub#05Am>22GL$syOHA(8G@y?P%EJrg# zopwIH|JyRPp&|=2#YTX@4gAfDyuvGRS8oy+kv~-4n4ROaa`$z7K80^!Ui$uN7Tq6l z*BQeL;2@)ym>ixQn;c~(2?BRmet5x0>5VTW3MIsc0+KQf87& zCaFIp@{$_}pj&bL}E&P`wAhabP>vtXL5 z>J(`-Xqr}#R*{w^>W168!Esu4RE-Q0$D!(%!)TdC?{azuwN{$BT~XySLiJ z!6wTz#d#Q+h(e&PUcUF`4>v6jP2YX?WCv`0S7!Ri-fg@1IG2mPij& zd*f@07qJqGp4nKn9g}9z$e9U`Y^FDpowkUbTz$5t{vc@Z*eSsqw>@hJZ%i<%+b8q@ zPyYt=7)Am%-?^P9VO=eNAsIXyMBF2Ru`2NWQSk&dVz42oV|M`~Flkkg(bRZ-%{rSu zlY6tQ_vBC?z@t+roRp+}WbSIJ(IE-7Hvxv6bfra^1tRN88BUH!;}ecHg=`9sd^W-C zmW6JDfdmUQB#|AWN|`C!J=s0gBPudSfTwS)xvYG^y)AUi{3c`?knK$mBYU1zPa z@``pz1t$YfGf?zmTM+|HDvRe?e)Ga%{Ocp*@IzMS^sNKg$P4`-$%osHxP%i)GmQcj~t4cy4#j&rrYdoq&-!Z$)P{rAI#`Gu^tEJlTO>(xS}3p$ksX-2AEU z{EM7Cy`mk>x=AlL&lOSk66mKw!_Tf4++N9myPw+@_uQ2&&yfZ=7@Nxz(#h%fV#v{> z02!qrBy68vaQNt-x3KT*O1&doB&Se+79mvow#LjTBx(7fXdWY_Qh{s`l9x6@S&q@} zAX3)T1t1i{Fp@s>R0x8UR3UbI&ibM8kuXG7-#Iw>fuL!<6KBmiQg<8g*EZHVth*WJ z^~@*QeDp-goClNHJ_Yzbq%|$fkC>&OwnR%&z55ElOi#@Zc*KkFla0MbW6=SZH=Cq# zQF+6j^T6{37osz22Iq>U$k8G`+cNr1w5}oviuC?LF!V{fM91)CAL7l9OnaUbVguki zDJcZn+``BItXtwlE(mK0VWu#iWkly>H4O@qOcme1`)`<%OvtchDlB~BP2*C})zuG) zcR|p){Z2k8hm)HdyV|=JAKBuCF5fosz@Oic=TZXqed#qj{;Z>H!gZ^LlHL(S;pRl_ zO@;RM-GBuG(EgSxb}R0D+(D)6RWgrN!kpVBo0aQ`V8Pf7FX3Br`_+oW6G|l91b06W z)bbO94|X&*MJjD_oY;-%32$+M!B4DRb(y}474yoq#an23R)!f&cecAVPXfTaFA`T6 z6Po|n?0QJE_@&%iAsSD3#k#A1m7jjZf53lQu>Xd^`b*sf#t2Svt`+U(lKverdfOBX z^(uvINmaAco5?5BQ(-_TOS>S`0nsp`ikKF$7I8T2;F`~88esS;*CB6E=2a~$oG0w< zQ+V687dY~QI--JpUj#30J4eLEKpJ_tEqwMS`G_xKaTWfSMBY5c&8NVrFMUC!nws^o zgJqvj4fZA`kwkL&1-lyIWuS6d?7e#gw>*k}OZm+H7WGaiCy~!4p$8D6Yma2vDF&m> z0cheRhk?76u9-*DGnNWj3Jc}T^E}5e$LS2Z+7+L*0xLdpv6uLs8eRWT722-3Y8NiR z?MoLav6flfmzZd}TZ`bRF`xLZ@|y2=I3 zLrV0RCLdHjly=z#Jy(%R8R|(nu~=Ecwh>9fFF2x%p)x_cTrYdQR21j4I-zF)p1 z(qHOf$Hz?uV)RX{9_$`WmC_}pT|lFDmr9pxfYLgB52Z2r;?4{CF@KGrpX^DBpOzL* zMVxv?-9b%jwTNcjFUnukCyOT!i=wuK^2qWsKmWX&VlT)Ug@+&Me8$d7P1D1O*NrpF z_eFdL5`K##1E_+FO~dpAPjFQzhuLUqn&O@AL!noYR`#v*YD*ytpNA}F#S`Z(cH?g` z`J@=+${(}USi8l)f>4_01Uix~sQ1ihe^ziF%Rr2|u3slwbeSS~r#_Vvw5Ft4a{VaW zPp$xqB%oKO;3S_0#>7b}(j;q!5cZ*%;~3qHZq=lzyI`yMkfK8ki&_MogrOYG(Qa#A z^W~GLJ{V3pL0VJ7S?A=jZ#qL~-c7g`Fc071#}bt5cz*I~+ES@;*RmKs-Nt@Jq%9Z5 zSKnhx3p#dr2u_?8}NtKaLe%?;ep+z#o}yC zQ8GfKt%#f#u&Zm)^waS(^Fe z1rAD9GEB8J+w>lBMA~@rc}|?bqsjNbaG<-tsMS}HA*U*j$6Mi#acElOf8A%daZHI2 zi}UuO-s(l*EoT?t#JNS_3lYum55tooLuUIw?WljkC|lW@`~i=*LtzvV?}EWHAl?3d zA+M-v_jQg4TYwP4k@NDQG7axJA%A?iQrS-h2Dj)jzpAOwaPCv2W-)o$!}*87nioIT z<)apUV9orIAf1mQ4M3g*Md?X8JOTg`NlYFO%PXSosl!h9M2UjnE9lsvM$EGv52__Zl=DzmIVSTo|ALurE?Q5L2ju^~r=Od0-8==rIvhWnk!9XP4^qpf}C zFKf~YzbdgdZT}2sV^eeX*f$UKkL5;K1G7V?4qMuRSd2w6-O7dDgtBD|B))5v< z!1owQSIa0u7w9n7evcR{=7Fr%Hc(He&QMR)A}kPQ@TyNl!9d5!t7ujiSBpKXlD`LJ zo_$0$im|NlW3C$15e6vQ4@K`f3+1UQ3^Gz{>T&Fo5u(uh+^C z<9OkQS$0dUYodna*g_RMRaK&l84^j+4H&>TF-7~5@xrPUsSHca$z0^0?zyVm<{j1r zQJ=X0F+2*@xLhYfidkKj>EvhqGfrYFEhrR)E8%`kLlb@PqoZ z?kvk-U1$UTyY=HdI@M6MrCH*4se~r8OFn8$jI_%6sMTV`9kj;erghlxh&Kl`Vn0z-Q8RCR^rO!(#W(Cq&C0HKDw+AYxsmeY2tl2)P%DKv$ zF3yESY(OY{_e)| z!hckL@eE;fYpkaGy(-vu+HYEAY&F~j7;JHkck99?HAvB^OnuIC!Gn~gVEOGfR7z6P zAyix(-O-E`6;@el-isTkSg%PG5+A?mkOcbJDA5_>8DWXfFh}WsDCKuz2qEe z)YM(ytFwi6;Q8u4!}wb=*gY7_6s(0@kY=iZRXrHWYv!~ZeS-kU_-bw{!r&tT&N%Uf zxVbg_o&@uK^b%2^ezAs&p+Fw1ga|_mTMN7XH+~EG2*3g~$3|f*1+M7-Djtt=c?|h& z(S+y+n^%UH_P%l>gJXnx9U5vb?k~a?ilgb`=(*z+1f26T0ER;{rWSPHLL_8VGkfICBe|g z_O&wi++QEc!*ovSqWtXs>*7ea%q0GIomS`$M(tP()nYQZX-xW-rbGX(YP%7 ze4we)t@0{d;j5F_O;YTXilp3p=_Up?+Vxa%cmk0jfi)=+?v(M$l}gw&vpC|53EfQZ zg`u=6ynH4bd!b?7=P<`OTJ~e}Ws7$ee&fPxm9{U>;zw2gwp*m-LyF4)8W)}6_=ear>w3% zTB~m#Z{grJl|H~;=q<_lQ~rmAwKdHF6<%rq|6^FQ$l4) zn_BA=Yndsotl`&s9jkZ?$k7NZS-iO6&|zTYoOkWz+{vvtG#ak@S(|$5>KGFVE;>50 zK$N^zPU|g>OW_aAB-XIg!f1O!nD0(S8iYo; z2Ds{2XNaECM$RV0zIfe=f8%y=ZnlugvQ?{9Y*iu(;QvbF@NhU-KK?1mlEu!Myk+V3 z^RQKsO4Qe*Vn#Hf-@xt?a8o?8!043pmbMM|d6l+dfAy$lXSDSJ)Ws5~54*a^>1>ly z?2cCtlM-Tn^0I^xyzU2k;z#L$)tTm{# zBk2pyMc@EQ3Y8w0{GdF3$7e^w2j{L@^0yT{ISP!}++|V=*R-ToMTs>6@+hG$B4m7C zX^66mxlN+>evq`EQk!lkFNjg0j6HSosG^o)V_)15(!0h>y1Uk2`ETe2=pFBFCNz}2 zpMB^4Xu00SUrkQO<<#3Ygytd_A35$Tb%(1Hs2@A~b}lyS|If zf1kOtQdUnh6Unxwe$SozjL;1dV?vPJr<<9RmyNjytg|J4L3)aKRcQH4FQHIeLC#H% zNzOHek4Z>_Pmf?t8iVu8pkNyRTl^>ZS?bR%Ui{?totZx0rHHJo-{WHx#|&(oco$@D z=sDk}zqDamm`m|_Gheqxr)lm`3C*WxolY_;(@*SR)fA;;ie`XN?)R(J$!QAHxS41z z6lj=gkV|*6jhh}Z)!9=W48LzRy1LaLD*8@POWzF&x7!yA_8d2jXtx!1CZPH|=y>&? zUL zuU|5}HTY%q;o8>;FOZ-cr*5nU($%pHG-i|Ma~~!y3mi(q3tbYK^yhXOV`7QC6C3oe zXb&E0vY{@Ab6|H--Ll@A8pg76iw1*G@k}dBcjg2%;(pC@3{HV^Fsu$bk2Y#MxxJ4& zTcVFJj;rsJ-xZG)7j;9UM!B5l+q76eo6ISG&0Va;AtaADuk84Q#hIe6MI>8O#!yOB zX6KQN16v-jHR7A98ts^i`ZW%z`a>W3G#)PL5V(Cx?r*UQP|uXX`WTje~5{ z7EzdYhhNcRYFv4R_~;jcOx58i49(hD{B$1h7e2A%y*!WwqFZCfj$6e6M0fx8vGDBN|;x&-&V~FA*RV7W%oi!5na|bklRIC{SXWvbo5~ zky7HJ>%}d0o8Qwy`|bn>b@aT@fY9~0MLRDh^i$(h$H_aV^uTm9ukik6(3U3C!dGZ? zw^>NoWplECEU6Z8%Umty?+OxrwsvaDY8KVi3$UlmetvbEbydF4<9@Z5rSWHyzdnC) z3$67C7q_3awk7&r%JbRos8UTp(8KQN{^OFrc{K!PY~KNy*fGa^qHNt*a@PhG>?>U5 zX7PKCklQYv*}=*-)yQMDk|~Gbr(vfBuKUS1{R%;k97=ZRdZ8wESf$UTC*`-N^A?`|*clU5wXvF;fjH)z0Iy^jFhqZfF z^Yht0A$|}K)lS(2r-$Nc6MQ!=xaTzj$3oeb>%_D55z(bdP#b6IPhhjV3x^400RH5t z030>wx=}?jtbv$dc=Fzs>Hsbk!&TbV8*Y=}L1X>JmBBt=L=C%4FVUY+6`L);fLTD< z&R1$Bz?XkIa~LGdgQ$v46}~KwfWbJ%;T8uUlAC21L-(hQv268GC^w(m z4uKc+B4J;*91T+TI{RfgGd<4HD40C%L==X0#=C5}vxG9oMLChE&&n5xst^Vnh__Im6u`_Wag1(z7!?HZ}Rx4FtEH~8N zmamZL$yXt7l>Or&+QEa4GE|)W$=RQ^-da*vCx{A z%#@w#7ncHmZ}}q&9}-yGWEzbVVRPwxHDtDHyZ=!PEufRpC4IA+YH+jqB? znvCS{(OZ9Ek)(ANkL4J9?EX{J{K8hCPhxqMQUN<3jWH^36n0X9uj*Ql;csLj!D&nk zQ97Z@=j+`(r{E=&vn0jl}N*d-JMQW6heHGPl(2l14I6koQ);Flq6 zg@hp{+eLxzn^VLj$dz7cYgAf*laWqyJPbiF{LjQ}^l1@O$40ngsFvdnj}Q+IB3&-G zWw^XZDb1rVV4*#lJ+Cq~ppmtajgYO7ReGN-+r$WRSxoec{n}gGw7ECxw`2C#LD8}{ ze@p!fse6L6<7=bAW4TONmFpR+lshCLk|<=f5xhw_Dj|`fF~`s5GJ|W@UUC8VU;st%byV_nVZ3i?&7IvoR`T zW82G|Tk^gk=MXiCW46@FX|w!9p|2b!H`U-4os5iuz$ixICG<4V&Sqy9BG%(Tiraqi zgT6A~e(&%3ws`AwMChg<$<-TNe&Yy#cz*cjw?k0Yh`d4I0x5e~%0T(ti}6jHi8uwb z<_(Lz0!>Pu_0$jRQ`prDb~5+Gob_jz;}J>B;Geh+P+O)La9}WP8M}L0Bcif6Wgz5x zFOSK@yQzFWZa-aj-7IEyMFX}0jCF>7$s#zFL9&E9x!R<1*1#5Ny>wRK3PyyZ49ZMJ zty1E&MBqpz5>@M5t3c-n(EbtA_Oj^Mh)@XY_3w-<{iAGTB_QhgZ&4Ctc~qjBZC7B_ z0euVf^Xw#!LF6rQq|RXetY74BF+IHq3HuT@3jGq#Ul;*$B~WI1QsEM(Kp&}22{^=N z2{Z-|bmRn|q&H75b+L^wrBFm>iS$^P+&3~h&q}IDT=D_zpLRIE>y)zSTHs46f%!U| zz1cL3T+ahsg$soBOL+`92GGsN3M%$Ef1hoOf&*J-dPP$?{zg@ z*LkkrU-$F9pYQ$L&-3r^^RRcNOEJT85Vl#S58DkkPlF;BR3cADkYo!K5t}-ZKI_Ru zWGdr&wR0ADTN?=>-?~I8d4KLmX1PZCwMt{i1|LDK_pEQLwu7<6F;OS@TA!-#P7X423`0TOk>zcA`^S_hpe0Vr3x(_=xjrgZ93}( zo$oYUTE{5)>PX(c6Pr5{T_duzi_9q}@C(i~ws?0_bJ#-na$k3zQXM+T1V1Hr_ zq-4B8V?pL+Qa_!-bdy+V9B?m(d?fR!rFmV0-CZT8oG}LTl|2D*@J~q1Sx)K_x*5H8 ztQ{7%zsgY&qZD-&JA5RqS!-lk>hgVMZ%O){a7Vx4`G^=n4QyVf=rf|Ly#M{sXm^L$ zYu#8ZOSOdhG)~QGjP!fUSd@5L|k+yucT`4I(WUG6ja5dr;IELXkE!PC5u z&5f5hnfF7-W;VYKqzP#E4G*jY=qi+Jv0o6QXdC#jiO-zkYk}Moef`*LLq$0IksPho zUC~*FsWzDKbDv0?r4#EDdh|L(BIgM~x0dFuw1N|a8eBxO2g&-hZ_uwCtHejB1M{Tb zV_&X4T3Z=hS(!j5SdP60`=gduR?vpuOS-Qh%SOo?!SZYR<8XFQ_hurU?6@(#LyQd+ zgA!Gbj{^tP1p3e1l#k#!*QXyT9+e>aV;|2UB+Y$8CYp?!{t?X1*uoIUZlg;zB&kQT zZVXTDI({try5H+e(D@qfSJD~9*yPw*({=sFv$JBTa3ywlU6mE6=z2NJv?gGr7eLF2j8dy1%Mb6+t^;D6uYFa7XN8_sYQdhXvx}A!>r>^tg3q)_ z?2hr`z}l6zahs8oR+7_W>vi%Tdkn=^L#Ug_b5{F1*-^~F+5)Z1fsQ9tsq>%b7iA0d zDza*`wib`h0W4ww7O_B-NzH&08ey;L{{p@NutW0CW%OA0x8PF8b}p&F{JILGb;2)5 zB`)CcV^7Pfp&IS;_E;LtH;aY~Hz7ySVuLzgI9(2Yvb*P1(U>ZhhQCZMN*@{}cC{on zjrn_cxtS!`mHC#Ex?ZKCM_Xjg5^|nz-I|JaO>8Hc$K<#-(iudP%BiBKT-m z%h9q^*_FO?)|9kdk1SgA4G!Lyh;-pSyT8+(KC&;Z?|eI;yHFXG^=Bm=c&1EE3d6mA z>m{$)B@Ml5pN0l7*iyM4;|8zV$3c@hu&t-IurU%+ z640ZYZ7MrjT^4D{s;`-Pi0_aiQl&4UW{XY*n|b9(w>x1b-x-CE2ZOzuS@;0$Al3(l zHmmd?)wT6|A*q|VKvxOiltiQM?yFbteoTkki{;B~6Z>mWxxpKm}W92E1sxKfic?ew4?{a)n|q(MUFc z=R$aJ==a;%J3lWJSfvPmf^q#F1NI0X&h?EXZsgfXNE`=RrXDVo=&~p`!meIwmuo;a zP7c3DzdRDQ^rn&vs%q|VqNT(&yv=mRu{=WqOwjv^Z=yZ5tHVmEiIq~4Ns#lr>>l&TuV())R2+yz9=xLN%?$e z9Ep(F7441)RB2pK?*UH9cKg)_^%9KFv`6HD)~{N0sI__8yMHOX6x8r(Pz zRd(S-d~L6xM0&!Aoa;WR>go>X$)aF~jo@)@mx>IgN0VI5;(gch=FGY=v=USut zB>-~w`n`;UhDM!O==VIo@rLvCl0e;VHONKV#qqXwD4?{`L8e&3jb&Q<`%2m*#9%R3 zdf~JK#&e3QSj6B2c$;U&^}UmO(9?ZjCaHKp?a2j+JViNXhjuEsEhR~-p!t@W&zNL9 z=}yeK)~1%G_c-i6FYiVnfy;Td?O=xDM0C^PM156j6sNNjj?^c2WSg4s*JI253i)-) z=|rOhuRXM_nnw2GS5rTLwxhoM-sS>G8L{g-c8Tclr`){GcFdvnm)MxLb!sgK@4~}z z_%1y8AlBla<9$*|$L~CE*|a_%PXpqqo9?$0JNKU?&TKi7d~W23zRsLliq{|J5-%S8 zF)(f=DST?3*|-5UZDCpnc()ziMhq(rBf@81e!5sT6dkhx1{e)G+3#S|H)sgY9HMzzzcsgKAPEFpR{*O@t$LZzuzZR!Vo9o`nGISIknh5 z%qYco@!4*#PpJdV01wk6jL<;HxjP<$dY2)ypoQ(RM#i5YM-Rujh?Vv{2@muWy{DJu+DCTMXYeJ1L8o#fTqx zTPJ&!`CMft2~pqs!Nb#m5xTsGM*1p&nO+LKeKM&g6foE;jUhwbMrWo5c152TV>`Q% z047t5ov*kNzhz|kedMLUJej#={=G!FP>01&HCs@@N>D8v9?ylmtIc~NR z&IHMIU75A3++BfWM@=AYY>p-V)0bxS2oe9g>cbZKPRG6S_hc7e2PUJ+`!6!YDAUc} z<{#$I_TDUMH51p}H{fCYfd89JZoE>nQaedYQ~Dv+!!@KN|r42l2=**kyDDqQ|w46Y4^ z{u~p+XdpsjkBANd))mo#pl}_C=z1`SkQpSJ3)h8;>fkV4knS%!xUP`x^JDy7A3_%b z74a3;2L;1_)`7LPVM3zLPq{)JLR-Wa41$88qQ1f`(H{I*NLM5m1VZRSMfV0mz;s|g z`wDe%A=l`qT0mf-2SOwU0zsi7H4AlMT~QBhunt_bFWO+Zj_7(iFp#$BdO9#2A>!%h zTA(nvwrCH)Al+Yj4+g=1_0@(#ME5J~{V%yXFoX~h^|P-I94rJ&{T&M=dI%X!L~$YF r%FZ{;`^VMQvLz4#6@DDXe>hyEKo4TzUpE`94H2HPxRR2o)#-l&X5~%^ From 3b3250339a12a5ec90923718ced3240d9728e841 Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 3 Feb 2015 18:15:01 -0800 Subject: [PATCH 284/374] Explicitely convert results of Bits Muxes to UInt Chisel updated to emit SInt result instead of UInt so this commit addresses this change. --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 794e3447..9040594b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -145,7 +145,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) From 7b86ea17cfc3dedb5989728847c8bf86493340ad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Feb 2015 19:37:32 -0800 Subject: [PATCH 285/374] rename L2HellaCache to L2HellaCacheBank --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9040594b..a033be2f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -328,7 +328,7 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends +class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) From 0c66e70f147273726f3b823249a7f20a520029af Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 6 Feb 2015 13:20:44 -0800 Subject: [PATCH 286/374] cleanup of conflicts; allocation bugfix --- uncore/src/main/scala/cache.scala | 13 ++++++------- uncore/src/main/scala/coherence.scala | 2 -- uncore/src/main/scala/tilelink.scala | 3 +++ uncore/src/main/scala/uncore.scala | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a033be2f..859e0bc3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -397,7 +397,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := acquire.valid && (acquire_idx === UInt(i)) + acq.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires @@ -475,7 +475,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val cacq = io.inner.acquire.bits val crel = io.inner.release.bits val cgnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) @@ -498,7 +497,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) io.has_release_match := !crel.payload.isVoluntary() && - co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + crel.payload.conflicts(xact_addr_block) && (state === s_probe) val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) @@ -824,12 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || - xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + cacq.payload.addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := xact.hasMultibeatData() && - (xact.addr_block === cacq.payload.addr_block) && + io.has_acquire_match := xact.conflicts(cacq.payload) && collect_cacq_data io.has_release_match := !crel.payload.isVoluntary() && (xact.addr_block === crel.payload.addr_block) && diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 201e67ab..82f72dea 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -135,8 +135,6 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3fc0e4e7..afbf4dc0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -49,6 +49,9 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) + + def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block + def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cc4755f0..1230771c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -300,10 +300,10 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && - (state != s_idle) && - !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + io.has_acquire_conflict := xact.conflicts(cacq.payload) && + !collect_inner_data && + (state != s_idle) + io.has_release_match := xact.conflicts(crel.payload) && !crel.payload.isVoluntary() && (state != s_idle) From 0a8722e881056c75ece113243f4544e702895997 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Feb 2015 00:35:18 -0800 Subject: [PATCH 287/374] bugfix for indexing DataArray of of small L2 --- uncore/src/main/scala/cache.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 859e0bc3..4380e255 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -281,9 +281,9 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasCacheBlockAddress with HasTileLinkBeatId with HasL2Id { + val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) } @@ -311,8 +311,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -524,7 +524,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) @@ -658,7 +658,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) @@ -881,12 +881,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) From 1bed6ea498de86d51815b0c8b95f861bbe8f6ad0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 28 Feb 2015 17:02:13 -0800 Subject: [PATCH 288/374] New metadata-based coherence API --- uncore/src/main/scala/broadcast.scala | 380 ++++++++ uncore/src/main/scala/cache.scala | 1236 ++++++++++++------------- uncore/src/main/scala/coherence.scala | 985 +++++++++----------- uncore/src/main/scala/directory.scala | 43 + uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/memserdes.scala | 314 +++++-- uncore/src/main/scala/metadata.scala | 188 ++++ uncore/src/main/scala/tilelink.scala | 307 +++--- uncore/src/main/scala/uncore.scala | 508 +++------- 9 files changed, 2117 insertions(+), 1851 deletions(-) create mode 100644 uncore/src/main/scala/broadcast.scala create mode 100644 uncore/src/main/scala/directory.scala create mode 100644 uncore/src/main/scala/metadata.scala diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala new file mode 100644 index 00000000..bd8a3f24 --- /dev/null +++ b/uncore/src/main/scala/broadcast.scala @@ -0,0 +1,380 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object L2StoreDataQueueDepth extends Field[Int] + +trait BroadcastHubParameters extends CoherenceAgentParameters { + val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation extends Bundle with BroadcastHubParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + +class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent + with BroadcastHubParameters { + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + // Create SHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + + // Propagate incoherence flags + trackerList.map(_.io.incoherent := io.incoherent.toBits) + + // Queue to store impending Put data + val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + + // Queue to store impending Voluntary Release data + val release = io.inner.release + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits + t.release.valid := release.valid && (release_idx === UInt(i)) + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Wire probe requests and grant reply to clients, finish acks from clients + // Note that we bypass the Grant data subbundles + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + { case TLId => params(OuterTLId) + case TLDataBits => internalDataBits }) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + // Get the pending data out of the store data queue + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.payload.hasData() && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } +} + +class BroadcastXactTracker extends XactTracker { + val io = new ManagerXactTrackerIO +} + +class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = ManagerMetadata.onReset + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + + io.has_acquire_conflict := Bool(false) + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + + io.outer.acquire.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + + //TODO: Use io.outer.release instead? + io.outer.acquire.bits.payload := Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(UInt(0)) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := Mux(io.irel().hasData(), s_outer, + Mux(io.irel().requiresAck(), s_ack, s_idle)) + } + } + is(s_outer) { + io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) + when(oacq_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { // Forward the Grant.voluntaryAck + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(io.inner.grant.fire()) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val coh = ManagerMetadata.onReset + + assert(!(state != s_idle && xact.isBuiltInType() && + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), + UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.incoherent.toBits | myflag) + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ognt_ack = Reg(init=Bool(false)) + val pending_outer_write = xact.hasData() + val pending_outer_read = io.ignt().hasData() + val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + + io.has_acquire_conflict := xact.conflicts(io.iacq()) && + !collect_iacq_data && + (state != s_idle) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state != s_idle) + io.has_acquire_match := Bool(false) + + val outer_write_acq = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + val outer_write_rel = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = io.irel().addr_beat, + data = io.irel().data))(outerTLParams) + val outer_read = Bundle(GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block))(outerTLParams) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default + io.outer.grant.ready := Bool(false) + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := coh.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + when(pending_ognt_ack) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } + //TODO add finish queue if this isnt the last level manager + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(UInt(0)) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + probe_flags := probe_initial_flags + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(pending_outer_write_, s_mem_write, + Mux(pending_outer_read_, s_mem_read, s_make_grant))) + } + } + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready + when(io.inner.release.valid) { + when(io.irel().hasData()) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } .otherwise { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_write) { // Write data to outer memory + io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.bits.payload := outer_write_acq + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + state := Mux(pending_outer_read, s_mem_read, s_mem_resp) + } + } + is(s_mem_read) { // Read data from outer memory (possibly what was just written) + io.outer.acquire.valid := !pending_ognt_ack + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.fire()) { state := s_mem_resp } + } + is(s_mem_resp) { // Wait to forward grants from outer memory + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(ignt_data_done) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4380e255..75eb897d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,7 +2,6 @@ package uncore import Chisel._ -import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -11,6 +10,7 @@ case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -140,7 +140,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(nWays){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) @@ -148,17 +147,14 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } + val metabits = rstVal.getWidth val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until nWays) { - val m = tags(metabits*(w+1)-1, metabits*w) - io.resp(w) := rstVal.clone.fromBits(m) - } - + io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } @@ -166,35 +162,17 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = tlDataBits/rowBits - val refillCycles = refillCyclesPerBeat*tlDataBeats + val blockAddrBits = params(TLBlockAddrBits) + val refillCyclesPerBeat = outerDataBits/rowBits + val refillCycles = refillCyclesPerBeat*outerDataBeats require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) - require(amoAluOperandBits <= tlDataBits) + require(amoAluOperandBits <= innerDataBits) + require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states } -abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters - -abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { - def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { - val (cnt, cnt_done) = - Counter(inc && data.hasMultibeatData(), tlDataBeats) - val done = (inc && !data.hasMultibeatData()) || cnt_done - (cnt, done) - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload)._2 - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { - connectDataBeatCounter(in.fire(), in.bits) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { - connectDataBeatCounter(in.valid, in.bits)._2 - } -} +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -206,22 +184,29 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } -trait HasL2Data extends HasTileLinkData { +trait HasL2BeatAddr extends L2HellaCacheBundle { + val addr_beat = UInt(width = log2Up(refillCycles)) +} + +trait HasL2Data extends L2HellaCacheBundle + with HasL2BeatAddr { + val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) + def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) +} + +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: ManagerMetadata) = { + def apply(tag: Bits, coh: HierarchicalMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new ManagerMetadata -} class L2MetaReadReq extends MetaReadReq with HasL2Id { val tag = Bits(width = tagBits) @@ -250,7 +235,8 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) + def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) + val meta = Module(new MetadataArray(onReset _)) meta.io.read <> io.read meta.io.write <> io.write @@ -259,7 +245,7 @@ class L2MetadataArray extends L2HellaCacheModule { def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) @@ -281,7 +267,7 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasTileLinkBeatId + with HasL2BeatAddr with HasL2Id { val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) @@ -289,7 +275,7 @@ class L2DataReadReq extends L2HellaCacheBundle class L2DataWriteReq extends L2DataReadReq with HasL2Data { - val wmask = Bits(width = tlWriteMaskBits) + val wmask = Bits(width = rowBits/8) } class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data @@ -305,7 +291,7 @@ trait HasL2DataWriteIO extends L2HellaCacheBundle { class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO -class L2DataArray extends L2HellaCacheModule { +class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip val wmask = FillInterleaved(8, io.write.bits.wmask) @@ -320,23 +306,22 @@ class L2DataArray extends L2HellaCacheModule { reg_raddr := raddr } - io.resp.valid := ShiftRegister(io.read.fire(), 1) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) - io.resp.bits.data := array(reg_raddr) + io.resp.valid := ShiftRegister(io.read.fire(), delay+1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) + io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) io.read.ready := !io.write.valid io.write.ready := Bool(true) } -class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - +class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent + with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) + val data = Module(new L2DataArray(1)) tshrfile.io.inner <> io.inner tshrfile.io.meta <> meta.io @@ -345,45 +330,28 @@ class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends io.incoherent <> tshrfile.io.incoherent } +class TSHRFileIO extends HierarchicalTLIO { + val meta = new L2MetaRWIO + val data = new L2DataRWIO +} -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta = new L2MetaRWIO - val data = new L2DataRWIO - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { - val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } - } +class TSHRFile(bankId: Int) extends L2HellaCacheModule + with HasCoherenceAgentWiringHelpers { + val io = new TSHRFileIO // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + Module(new L2VoluntaryReleaseTracker(id, bankId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + Module(new L2AcquireTracker(id, bankId)) } - val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire @@ -414,29 +382,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(releaseList.map(_.ready)).read(release_idx) - // Wire finished transaction acks - val finish = io.inner.finish - val finish_idx = finish.bits.payload.manager_xact_id - trackerList.zipWithIndex.map { case (t, i) => - t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) - } - trackerList.map(_.io.inner.finish.bits := finish.bits) - finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - - // Wire probe requests to clients + // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) - - // Wire grant reply to initiating client - doOutputArbitration( - io.inner.grant, - trackerList.map(_.io.inner.grant), - tlDataBeats, - (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), - {case TLId => outerId}) + val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -449,10 +402,495 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } + +class L2XactTrackerIO extends HierarchicalXactTrackerIO { + val data = new L2DataRWIO + val meta = new L2MetaRWIO + val wb = new L2WritebackIO +} + +abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { + if(data.refillCycles > 1) { + val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) + } else { (UInt(0), inc) } + } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + in: DecoupledIO[T], + beat: UInt = UInt(0), + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.fire(), in.bits, beat, full_block) + } + def connectInternalDataBeatCounter[T <: HasL2Data]( + in: ValidIO[T], + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 + } +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = xact_meta.coh + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := io.irel().isVoluntary() + + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.wb.req.valid := Bool(false) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(io.irel().addr_beat) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := Mux(io.meta.resp.bits.tag_match, + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_inner_grant, s_idle)) + } + } + is(s_data_write) { + io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? + } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + Reg(io.iacq().data.clone) + } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val pending_coh = Reg{ xact_meta.coh.clone } + val pending_finish = Reg{ io.outer.finish.bits.clone } + + val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val do_allocate = xact.allocate() + val needs_writeback = !xact_tag_match && do_allocate && + xact_meta.coh.outer.requiresVoluntaryWriteback() + val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src), + pending_coh.outer.onHit(M_XWR)) // WB is a write + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.ognt(), xact.op_code())) + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val mask_self = Mux(xact.requiresSelfProbe(), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := data_buffer(0) // default + + // TODO: figure out how to merge the following three versions of this func + def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) + def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + + //TODO: Allow hit under miss for stores + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + io.iacq().addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := !io.irel().isVoluntary() && + (xact.addr_block === io.irel().addr_block) && + (state === s_probe) + + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Mux(do_allocate, + xact_meta.coh.outer.makeAcquire( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_cnt, + data = Mux(xact.is(Acquire.putAtomicType), + data_buffer(innerDataBeats), + data_buffer(ignt_data_cnt))) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.read.bits.addr_beat := read_data_cnt + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh := pending_coh + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(io.iacq().addr_beat) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + irel_had_data := Bool(false) + ognt_had_data := Bool(false) + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + pending_coh := io.meta.resp.bits.meta.coh + val _coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match + val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _needs_writeback = !_tag_match && do_allocate && + _coh.outer.requiresVoluntaryWriteback() + val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + when(_is_hit) { pending_coh := pending_coh_on_hit } + when(_needs_probes) { + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + } + state := Mux(_tag_match, + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + } + } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_acquire } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := pending_probes != UInt(0) + when(io.inner.probe.ready) { + pending_probes := pending_probes & ~UIntToOH(curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + pending_coh := pending_coh_on_irel + // Handle released dirty data + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! + when(io.irel().hasData()) { + irel_had_data := Bool(true) + mergeDataInner(data_buffer, io.irel()) + } + // We don't decrement release_count until we've received all the data beats. + when(!io.irel().hasMultibeatData() || irel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + } + } + is(s_outer_acquire) { + io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + when(oacq_data_done) { + state := s_outer_grant + } + } + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().hasData()) { + mergeDataOuter(data_buffer, io.ognt()) + ognt_had_data := Bool(true) + } + when(ognt_data_done) { + pending_coh := pending_coh_on_ognt + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := Mux(!do_allocate, s_inner_grant, + Mux(io.ognt().hasData(), s_data_write, s_data_read)) + } + } + } + } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { + state := Mux(!do_allocate, s_inner_grant, + Mux(ognt_had_data, s_data_write, s_data_read)) + } + } + is(s_data_read) { + io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(read_data_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(resp_data_done) { + state := Mux(xact.hasData(), s_data_write, s_inner_grant) + } + } + is(s_data_write) { + io.data.write.valid := Bool(true) + when(write_data_done) { state := s_inner_grant } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(ignt_data_done) { + val meta = pending_coh_on_ignt != xact_meta.coh + when(meta) { pending_coh := pending_coh_on_ignt } + state := Mux(meta, s_meta_write, + Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr_block = UInt(width = tlBlockAddrBits) - val coh = new ManagerMetadata + val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size + val coh = new HierarchicalMetadata val way_en = Bits(width = nWays) } @@ -463,58 +901,63 @@ class L2WritebackIO extends L2HellaCacheBundle { val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val wb = new L2WritebackIO().flip - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - } - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val mgnt = io.outer.grant.bits +class L2WritebackUnitIO extends HierarchicalXactTrackerIO { + val wb = new L2WritebackIO().flip + val data = new L2DataRWIO +} - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) +class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2WritebackUnitIO + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_coh = Reg{ new ManagerMetadata } + val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) + val xact_coh = Reg{ new HierarchicalMetadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } + val pending_finish = Reg{ io.outer.finish.bits.clone } - val crel_had_data = Reg(init = Bool(false)) + val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) - val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - io.has_release_match := !crel.payload.isVoluntary() && - crel.payload.conflicts(xact_addr_block) && + val pending_icoh_on_irel = xact_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := !io.irel().isVoluntary() && + io.irel().conflicts(xact_addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) // default + io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( client_xact_id = UInt(trackerId), addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - io.outer.grant.ready := Bool(false) // Never gets mgnts + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) + io.outer.release.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // default + io.outer.finish.valid := Bool(false) // default + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) + io.inner.probe.bits.payload := + xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -540,528 +983,75 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id + irel_had_data := Bool(false) val coh = io.wb.req.bits.coh - val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - pending_probes := mask_incoherent - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) + val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) } state := Mux(needs_probes, s_probe, s_data_read) } } is(s_probe) { // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) + io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_coh := next_coh_on_rel + xact_coh.inner := pending_icoh_on_irel // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - xact_data(crel.payload.addr_beat) := crel.payload.data + when(io.irel().hasData()) { + irel_had_data := Bool(true) + data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasData() || crel_data_done) { + when(!io.irel().hasData() || irel_data_done) { release_count := release_count - UInt(1) } } when(release_count === UInt(0)) { - state := Mux(crel_had_data, s_outer_write, s_data_read) + state := Mux(irel_had_data, s_outer_release, s_data_read) } } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_write } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_release } } - is(s_outer_write) { - io.outer.acquire.valid := Bool(true) - when(macq_data_done) { - io.wb.resp.valid := Bool(true) - state := s_idle + is(s_outer_release) { + io.outer.release.valid := Bool(true) + when(orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) } } - } -} - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - val meta = new L2MetaRWIO - val wb = new L2WritebackIO - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cack = io.inner.finish.bits - val mgnt = io.outer.grant.bits -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_crel_data = Reg(init=Bool(false)) - val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) - - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, - xact_meta.coh, - xact_src) - io.wb.req.valid := Bool(false) - - when(collect_crel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(crel.payload.addr_beat) := crel.payload.data - crel_data_valid(crel.payload.addr_beat) := Bool(true) - } - when(crel_data_done) { collect_crel_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_addr_beat := crel.payload.addr_beat - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_crel_data := crel.payload.hasMultibeatData() - crel_data_valid := Bits(1) - state := s_meta_read + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := s_wb_resp + } } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } -} - - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result - Reg(io.inner.acquire.bits.payload.data.clone) - } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) - - val collect_cacq_data = Reg(init=Bool(false)) - val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) - val curr_p_id = co.dir.next(pending_probes) - val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) - val mask_incoherent = mask_self & ~io.tile_incoherent - - val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - - val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit - val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) - val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_builtin_type || xact.allocate() - - val amoalu = Module(new AMOALU) - amoalu.io.addr := xact.addr() - amoalu.io.cmd := xact.op_code() - amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := xact.data(0) // default - - def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.uncachedAtomic), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = tlDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result - } - - //TODO: Allow hit under miss for stores - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - cacq.payload.addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && - (state != s_idle) && - !collect_cacq_data - io.has_acquire_match := xact.conflicts(cacq.payload) && - collect_cacq_data - io.has_release_match := !crel.payload.isVoluntary() && - (xact.addr_block === crel.payload.addr_block) && - (state === s_probe) - - val next_coh_on_rel = co.managerMetadataOnRelease( - incoming = crel.payload, - meta = xact_meta.coh, - src = crel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant( - outgoing = cgnt.payload, - meta = xact_meta.coh, - dst = cgnt.header.dst) - - val outer_write = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := Bool(true) //grant.data -> xact.data - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant( - manager_xact_id = UInt(trackerId), - meta = xact_meta.coh, - addr_beat = cgnt_data_cnt, - data = Mux(xact.is(Acquire.uncachedAtomic), - xact_data(tlDataBeats), - xact_data(cgnt_data_cnt))) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_gnt - - io.wb.req.valid := Bool(false) - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh - io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) - - when(collect_cacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(cacq.payload.addr_beat) := cacq.payload.data - cacq_data_valid(cacq.payload.addr_beat) := Bool(true) - } - when(cacq_data_done) { collect_cacq_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - when( io.inner.acquire.valid ) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_cacq_data := cacq.payload.hasMultibeatData() - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - val coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit - val _needs_probes = _tag_match && co.requiresProbes(xact, coh) - val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.builtin_type || - xact.hasMultibeatData() || - cgnt.payload.hasMultibeatData() - read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) - write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) - resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) - when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) - } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe - } - } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_read } - } - is(s_probe) { - // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) - when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) - } - // Handle releases, which may have data being written back - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_rel - // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - mergeData(xact_data, crel.payload) - } - // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasMultibeatData() || crel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) - } - } - is(s_outer_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := s_outer_resp - } - } - is(s_outer_resp) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - //TODO make sure cacq data is actually present before merging - mergeData(xact_data, mgnt.payload) - when(mgnt_data_done) { - state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) - } - } - } - is(s_data_read) { - io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(io.data.read.ready) { - read_data_cnt := read_data_cnt + UInt(1) - when(read_data_cnt === read_data_max) { state := s_data_resp } - } - } - is(s_data_resp) { - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(resp_data_cnt === resp_data_max) { - state := Mux(xact.hasData(), s_data_write, s_meta_write) - } - } - is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { - write_data_cnt := write_data_cnt + UInt(1) - when(write_data_cnt === write_data_max) { - state := s_meta_write - } - } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - cgnt_data_cnt := cgnt_data_cnt + UInt(1) - when(cgnt_data_cnt === cgnt_data_max) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - } - is(s_ack) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + io.wb.resp.valid := Bool(true) + state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 82f72dea..2524f3d1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,202 +3,148 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers -abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +/** The entire CoherencePolicy API consists of the following three traits: + * HasCustomTileLinkMessageTypes, used to define custom messages + * HasClientSideCoherencePolicy, for client coherence agents + * HasManagerSideCoherencePolicy, for manager coherence agents + */ +abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends + HasCustomTileLinkMessageTypes with + HasClientSideCoherencePolicy with + HasManagerSideCoherencePolicy -class ClientMetadata extends CoherenceMetadata { - val state = UInt(width = co.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata().asInstanceOf[this.type] -} -object ClientMetadata { - def apply(state: UInt) = { - val meta = new ClientMetadata - meta.state := state - meta - } -} - -class ManagerMetadata extends CoherenceMetadata { - val state = UInt(width = co.masterStateWidth) - val sharers = UInt(width = co.dir.width) - override def clone = new ManagerMetadata().asInstanceOf[this.type] -} -object ManagerMetadata { - def apply(state: UInt, sharers: UInt) = { - val meta = new ManagerMetadata - meta.state := state - meta.sharers := sharers - meta - } - def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) -} - -// This class encapsulates transformations on different directory information -// storage formats -abstract class DirectoryRepresentation(val width: Int) { - def pop(prev: UInt, id: UInt): UInt - def push(prev: UInt, id: UInt): UInt - def flush: UInt - def none(s: UInt): Bool - def one(s: UInt): Bool - def count(s: UInt): UInt - def next(s: UInt): UInt - def full(s: UInt): UInt -} - -class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { - def pop(prev: UInt, id: UInt) = UInt(0) - def push(prev: UInt, id: UInt) = UInt(0) - def flush = UInt(0) - def none(s: UInt) = Bool(false) - def one(s: UInt) = Bool(false) - def count(s: UInt) = UInt(nClients) - def next(s: UInt) = UInt(0) - def full(s: UInt) = SInt(-1, width = nClients).toUInt -} - -class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { - def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) - def push(prev: UInt, id: UInt) = prev | UIntToOH(id) - def flush = UInt(0, width = width) - def none(s: UInt) = s === UInt(0) - def one(s: UInt) = PopCount(s) === UInt(1) - def count(s: UInt) = PopCount(s) - def next(s: UInt) = PriorityEncoder(s) - def full(s: UInt) = s -} - -// Coherence policy inferface for clients and managers -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { - def nClientStates: Int - def nManagerStates: Int - def nAcquireTypes: Int - def nProbeTypes: Int - def nReleaseTypes: Int - def nGrantTypes: Int - def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nManagerStates) +/** This API defines the custom, coherence-policy-defined message types, + * as opposed to the built-in ones found in tilelink.scala. + * Policies must enumerate the custom messages to be sent over each + * channel, as well as which of them have associated data. + */ +trait HasCustomTileLinkMessageTypes { + val nAcquireTypes: Int def acquireTypeWidth = log2Up(nAcquireTypes) + val nProbeTypes: Int def probeTypeWidth = log2Up(nProbeTypes) + val nReleaseTypes: Int def releaseTypeWidth = log2Up(nReleaseTypes) + val nGrantTypes: Int def grantTypeWidth = log2Up(nGrantTypes) - val clientStatesWithReadPermission: Vec[UInt] - val clientStatesWithWritePermission: Vec[UInt] - val clientStatesWithDirtyData: Vec[UInt] val acquireTypesWithData = Nil // Only built-in Acquire types have data for now val releaseTypesWithData: Vec[UInt] val grantTypesWithData: Vec[UInt] +} +/** This API contains all functions required for client coherence agents. + * Policies must enumerate the number of client states and define their + * permissions with respect to memory operations. Policies must fill in functions + * to control which messages are sent and how metadata is updated in response + * to coherence events. These funtions are generally called from within the + * ClientMetadata class in metadata.scala + */ +trait HasClientSideCoherencePolicy { + // Client coherence states and their permissions + val nClientStates: Int + def clientStateWidth = log2Ceil(nClientStates) + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + + // Transaction initiation logic def isValid(meta: ClientMetadata): Bool - def isValid(meta: ManagerMetadata): Bool - def isHit(cmd: UInt, meta: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), clientStatesWithWritePermission.contains(meta.state), clientStatesWithReadPermission.contains(meta.state)) } - //TODO: Use outer protocol's clientState instead, remove this function: - def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + //TODO: Assumes all states with write permissions also have read permissions + def requiresAcquireOnSecondaryMiss( + first_cmd: UInt, + second_cmd: UInt, + meta: ClientMetadata): Bool = { + isWriteIntent(second_cmd) && + !isWriteIntent(first_cmd) || + (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: - def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = clientStatesWithDirtyData.contains(meta.state) - def needsWriteback(meta: ClientMetadata): Bool = - needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + // Determine which custom message type to use + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + + // Mutate ClientMetadata based on messages or cmds + def clientMetadataOnReset: ClientMetadata def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnFlush: ClientMetadata - - def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnFlush: ManagerMetadata - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt - def getReleaseType(p: Probe, meta: ClientMetadata): UInt - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt - - def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata } -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 2 - def nManagerStates = 2 - def nAcquireTypes = 1 - def nProbeTypes = 2 - def nReleaseTypes = 4 - def nGrantTypes = 1 +/** This API contains all functions required for manager coherence agents. + * Policies must enumerate the number of manager states. Policies must fill + * in functions to control which Probe and Grant messages are sent and how + * metadata should be updated in response to coherence events. These funtions + * are generally called from within the ManagerMetadata class in metadata.scala + */ +trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { + val nManagerStates: Int + def masterStateWidth = log2Ceil(nManagerStates) - val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Transaction probing logic + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + + // Determine which custom message type to use in response + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + + // Mutate ManagerMetadata based on messages or cmds + def managerMetadataOnReset: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached + meta.sharers, + dir.push(meta.sharers, dst))) + //state = meta.state) TODO: Fix 0-width wires in Chisel +} + +/** The following concrete implementations of CoherencePolicy each provide the + * functionality of one particular protocol. + */ + +/** A simple protocol with only two Client states. + * Data is always assumed to be dirty. + * Only a single client may ever have a copy of a block at a time. + */ +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 1 + val nProbeTypes = 2 + val nReleaseTypes = 4 + val nGrantTypes = 1 val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientValid) - val clientStatesWithWritePermission = Vec(clientValid) - val clientStatesWithDirtyData = Vec(clientValid) val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 2 + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (outstanding.a_type != acquireExclusive) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata(Mux(incoming.p_type === probeInvalidate, - clientInvalid, meta.state)) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) @@ -213,112 +159,85 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A simple protocol with only three Client states. + * Data is marked as dirty when written. + * Only a single client may ever have a copy of a block at a time. + */ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 1 - - val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 1 val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def isValid (meta: ClientMetadata) = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -333,75 +252,113 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), Bool(true)))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A protocol with only three Client states. + * Data is always assumed to be dirty. + * Multiple clients may share read permissions on a block at the same time. + */ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantShared, grantExclusive) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid(meta: ManagerMetadata) = meta.state != managerInvalid + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -409,9 +366,9 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -424,30 +381,82 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) } +} - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) +/** A protocol with four Client states. + * Data is marked as dirty when written. + * Multiple clients may share read permissions on a block at the same time. + */ +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + // Client states and functions + val nClientStates = 4 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -462,80 +471,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) - - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = - MuxLookup(cmd, probeCopy, Array( - M_FLUSH -> probeInvalidate, - M_PRODUCE -> probeDowngrade)) - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) - - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) -} - -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 4 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - - val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) - - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -544,13 +484,12 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { clientShared, meta.state), M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, - grantExclusive -> Mux(outstanding.a_type === acquireExclusive, - clientExclusiveDirty, clientExclusiveClean), + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty)))) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = @@ -560,188 +499,80 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) - MuxLookup(cmd, releaseCopyAck, Array( - M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), - M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), - M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) - } - - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = - MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 7 - def nManagerStates = 2 - def nAcquireTypes = 3 - def nProbeTypes = 4 - def nReleaseTypes = 10 - def nGrantTypes = 4 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 3 + val nProbeTypes = 4 + val nReleaseTypes = 10 + val nGrantTypes = 4 val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + // Client states and functions + val nClientStates = 7 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), - meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), - clientShared, meta.state), - M_CLEAN -> MuxLookup(meta.state, meta.state, Array( - clientExclusiveDirty -> clientExclusiveClean, - clientMigratoryDirty -> clientMigratoryClean))))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantShared -> clientShared, - grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireExclusive -> clientExclusiveDirty, - acquireShared -> clientExclusiveClean)), - grantExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireExclusive -> clientMigratoryDirty, - acquireShared -> clientMigratoryClean)))))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> meta.state, - probeDowngrade -> MuxLookup(meta.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid))))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next, - r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next)) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), - Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), - outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -769,43 +600,89 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(dirty, with_data, without_data) } - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - MuxLookup(a.a_type, grantShared, Array( - acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - acquireExclusive -> grantExclusive, - acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers))) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped, + incoming.is(releaseInvalidateDataMigratory) -> popped, + incoming.is(releaseInvalidateAckMigratory) -> popped)) + } } diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/directory.scala new file mode 100644 index 00000000..db555ad3 --- /dev/null +++ b/uncore/src/main/scala/directory.scala @@ -0,0 +1,43 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// This class encapsulates transformations on different directory information +// storage formats +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt + def full(s: UInt): UInt +} + +abstract trait HasDirectoryRepresentation { + val dir: DirectoryRepresentation +} + +class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(nClients) + def next(s: UInt) = UInt(0) + def full(s: UInt) = SInt(-1, width = nClients).toUInt +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s +} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5e5053be..4d5e1135 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) val dataBeats = params(TLDataBeats) - val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) val offsetBits = params(HTIFOffsetBits) @@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWriteBlock( + PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), - UncachedReadBlock(addr_block = init_addr)) + GetBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid @@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e58dd08d..f10b3257 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } +//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary + + //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemIO } - val co = params(TLCoherence) val mifTagBits = params(MIFTagBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) @@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + require(params(TLClientXactIdBits) <= params(MIFTagBits)) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + io.tl.grant <> gnt_arb.io.out + + val acq_has_data = io.tl.acquire.bits.payload.hasData() + val rel_has_data = io.tl.release.bits.payload.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - - io.uncached.acquire.ready := Bool(false) - io.uncached.grant.valid := Bool(false) - io.mem.resp.ready := Bool(false) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - - //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are - //acceptable Acquire types - val acq_has_data = io.uncached.acquire.bits.payload.hasData() - val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tl_done_out = Reg(init=Bool(false)) - val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + val grant_for_rel = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = tag_out, + manager_xact_id = UInt(0)) + val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_out(0), + a_type = tag_out >> UInt(1), + client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) - when(!active_out){ - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr_block - has_data := acq_has_data - tl_done_out := tl_wrap_out - mif_done_out := Bool(false) - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.tl.release.bits.payload.client_xact_id + addr_out := io.tl.release.bits.payload.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + addr_out := io.tl.acquire.bits.payload.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + } } - } - when(active_out) { - when(!cmd_sent_out) { - mem_cmd_q.io.enq.valid := Bool(true) - } - when(has_data) { - when(!tl_done_out) { - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out } } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) } } - when(mem_cmd_q.io.enq.fire()) { - cmd_sent_out := Bool(true) + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tag_out := io.mem.req_cmd.bits.tag + addr_out := io.mem.req_data.bits.data + has_data := io.mem.req_cmd.bits.rw + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + io.mem.req_cmd.bits.rw := rel_has_data + io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + io.mem.req_cmd.bits.rw := acq_has_data + io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + } + } } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || mif_done_out)) { - active_out := Bool(false) + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } } } - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - // Aggregate incoming MemIO responses into TL Grants - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) val active_in = Reg(init=Bool(false)) - val mif_done_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) - val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - - when(active_in) { - io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_in(0), + a_type = tag_in >> UInt(1), + client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { io.mem.resp.ready := Bool(true) when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data } } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits.payload := + ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = io.mem.resp.bits.tag(0), + a_type = io.mem.resp.bits.tag >> UInt(1), + client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } - - io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), - g_type = Grant.uncachedReadBlock, - client_xact_id = tag_in, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemPipeIO } - val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val a = Module(new MemIOTileLinkIOConverter(1)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) - a.io.uncached <> io.uncached + a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala new file mode 100644 index 00000000..8f5d7e90 --- /dev/null +++ b/uncore/src/main/scala/metadata.scala @@ -0,0 +1,188 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// Classes to represent coherence information in clients and managers +abstract class CoherenceMetadata extends Bundle { + val co = params(TLCoherencePolicy) + val id = params(TLId) +} + +/* The ClientMetadata stores the client-side coherence information, + such as permissions on the data and whether the data is dirty. + Its API can be used to make TileLink messages in response to + memory operations or TileLink Probes. +*/ +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + + def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + + def isValid(dummy: Int = 0): Bool = co.isValid(this) + def isHit(cmd: UInt): Bool = co.isHit(cmd, this) + def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) + def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) + def requiresReleaseOnCacheControl(cmd: UInt): Bool = + co.requiresReleaseOnCacheControl(cmd: UInt, this) + def requiresVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresReleaseOnCacheControl(M_FLUSH, this) + + def makeAcquire( + client_xact_id: UInt, + addr_block: UInt, + op_code: UInt): Acquire = { + Bundle(Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true))), + { case TLId => id }) + } + + def makeVoluntaryWriteback( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def makeRelease( + prb: Probe, + client_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(false), + r_type = co.getReleaseType(prb, this), + client_xact_id = client_xact_id, + addr_block = prb.addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onGrant(incoming: Grant, pending: UInt): ClientMetadata = + Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + def onProbe(incoming: Probe): ClientMetadata = + Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + def onHit(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) + def onCacheControl(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) +} + +object ClientMetadata { + def apply(state: UInt) = { + val meta = new ClientMetadata + meta.state := state + meta + } + def onReset = new ClientMetadata().co.clientMetadataOnReset +} + +/* The ManagerMetadata stores manager-side information about the status + of a cache block, including whether it has any known sharers. Its + API can be used to create Probe and Grant TileLink messages. +*/ +class ManagerMetadata extends CoherenceMetadata { + // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + val sharers = UInt(width = co.dir.width) + + def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel + this.sharers === rhs.sharers + def full(dummy: Int = 0) = co.dir.full(this.sharers) + + def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresProbes(M_FLUSH, this) + + def makeProbe(cmd: UInt, addr_block: UInt): Probe = + Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + + def makeProbe(acq: Acquire): Probe = + Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + + def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = + makeProbe(M_FLUSH, addr_block) + + def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + Bundle(Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = rel.client_xact_id, + manager_xact_id = manager_xact_id), { case TLId => id }) + } + + def makeGrant( + acq: Acquire, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Bundle(Grant( + is_builtin_type = acq.isBuiltInType(), + g_type = Mux(acq.isBuiltInType(), + acq.getBuiltInGrantType(), + co.getGrantType(acq, this)), + client_xact_id = acq.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onRelease(incoming: Release, src: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + + def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) +} + +object ManagerMetadata { + def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + val meta = new ManagerMetadata + //meta.state := state TODO: Fix 0-width wires in Chisel + meta.sharers := sharers + meta + } + def apply() = { + val meta = new ManagerMetadata + //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel + meta.sharers := meta.co.dir.flush + meta + } + def onReset = new ManagerMetadata().co.managerMetadataOnReset +} + +/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + that is a Manager with respect to some inner caches and a Client with + respect to some outer cache. +*/ +class HierarchicalMetadata extends CoherenceMetadata { + val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) + val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) + def ===(rhs: HierarchicalMetadata): Bool = + this.inner === rhs.inner && this.outer === rhs.outer + def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) +} + +object HierarchicalMetadata { + def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + val m = new HierarchicalMetadata + m.inner := inner + m.outer := outer + m + } + def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) +} + +case object InnerTLId extends Field[String] +case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index afbf4dc0..c8996ddf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -8,30 +8,35 @@ import scala.math.max // external requirements or design space exploration // case object TLId extends Field[String] // Unique name per network -case object TLCoherence extends Field[CoherencePolicy] -case object TLAddrBits extends Field[Int] +case object TLCoherencePolicy extends Field[CoherencePolicy] +case object TLBlockAddrBits extends Field[Int] case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] +case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) + val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = M_SZ - val tlUncachedOperandSizeBits = MT_SZ - val tlSubblockUnionBits = max(tlWriteMaskBits, + val tlMemoryOpcodeBits = M_SZ + val tlMemoryOperandSizeBits = MT_SZ + val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes), + tlCoh.acquireTypeWidth) + val tlAcquireUnionBits = max(tlWriteMaskBits, (tlByteAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 - val co = params(TLCoherence) - val networkPreservesPointToPointOrdering = false //TODO: check physical network type + tlMemoryOperandSizeBits + + tlMemoryOpcodeBits)) + 1 + val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), + tlCoh.grantTypeWidth) + 1 + val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) - def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block - def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr + def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block + def conflicts(addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { @@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -abstract trait HasTileLinkData extends HasTileLinkBeatId { +trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } @@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val builtin_type = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) - val subblock = Bits(width = tlSubblockUnionBits) + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) + val union = Bits(width = tlAcquireUnionBits) // Utility funcs for accessing subblock union - val opSizeOff = tlByteAddrBits + 1 - val opCodeOff = tlUncachedOperandSizeBits + opSizeOff - val opMSB = tlAtomicOpcodeBits + opCodeOff - def allocate(dummy: Int = 0) = subblock(0) - def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) - def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) - def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) - def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + val opCodeOff = 1 + val opSizeOff = tlMemoryOpcodeBits + opCodeOff + val addrByteOff = tlMemoryOperandSizeBits + opSizeOff + val addrByteMSB = tlByteAddrBits + addrByteOff + def allocate(dummy: Int = 0) = union(0) + def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) + def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + + // Assumes no custom types have data + def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType - def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = - Probe(co.getProbeType(this, meta), this.addr_block) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { - Grant( - builtin_type = this.builtin_type, - g_type = co.getGrantType(this, meta), - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id, - addr_beat = addr_beat, - data = data - ) + def getBuiltInGrantType(dummy: Int = 0): UInt = { + MuxLookup(this.a_type, Grant.ackType, Array( + Acquire.getType -> Grant.dataBeatType, + Acquire.getBlockType -> Grant.dataBlockType, + Acquire.putType -> Grant.ackType, + Acquire.putBlockType -> Grant.ackType, + Acquire.putAtomicType -> Grant.dataBeatType)) } } object Acquire { - val nBuiltinAcquireTypes = 5 + val nBuiltInTypes = 5 //TODO: Use Enum - def uncachedRead = UInt(0) - def uncachedReadBlock = UInt(1) - def uncachedWrite = UInt(2) - def uncachedWriteBlock = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) - def typesWithMultibeatData = Vec(uncachedWriteBlock) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock - def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock + def getType = UInt("b000") + def getBlockType = UInt("b001") + def putType = UInt("b010") + def putBlockType = UInt("b011") + def putAtomicType = UInt("b100") + def typesWithData = Vec(putType, putBlockType, putAtomicType) + def typesWithMultibeatData = Vec(putBlockType) + def typesOnSubBlocks = Vec(putType, getType, putAtomicType) def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt // Most generic constructor def apply( - builtin_type: Bool, + is_builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0), - subblock: UInt = UInt(0)): Acquire = { + union: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.builtin_type := builtin_type + acq.is_builtin_type := is_builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block acq.addr_beat := addr_beat acq.data := data - acq.subblock := subblock + acq.union := union acq } - // For cached types - def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { - apply( - builtin_type = Bool(false), - a_type = a_type, - client_xact_id = client_xact_id, - addr_block = addr_block) - } // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -177,58 +169,73 @@ object Acquire { } // Asks for a single TileLink beat of data -object UncachedRead { +object Get { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedRead, + is_builtin_type = Bool(true), + a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - subblock = alloc) + union = Cat(M_XRD, alloc)) } } // Asks for an entire cache block of data -object UncachedReadBlock { +object GetBlock { def apply( client_xact_id: UInt = UInt(0), addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedReadBlock, + is_builtin_type = Bool(true), + a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - subblock = alloc.toUInt) + union = Cat(M_XRD, alloc)) } } -object UncachedWrite { +// Writes up to a single TileLink beat of data, using mask +object Put { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask, - alloc: Bool = Bool(true)): Acquire = { + write_mask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWrite, + is_builtin_type = Bool(true), + a_type = Acquire.putType, addr_block = addr_block, addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - subblock = Cat(write_mask, alloc)) + union = Cat(write_mask, Bool(true))) } } -// For full block of data -object UncachedWriteBlock { +// Writes an entire cache block of data +object PutBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + } def apply( client_xact_id: UInt, addr_block: UInt, @@ -236,17 +243,18 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWriteBlock, + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(Acquire.fullWriteMask, alloc)) + union = Cat(Acquire.fullWriteMask, alloc)) } } -object UncachedAtomic { +// Performs an atomic operation in the outer memory +object PutAtomic { def apply( client_xact_id: UInt, addr_block: UInt, @@ -256,48 +264,30 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedAtomic, + is_builtin_type = Bool(true), + a_type = Acquire.putAtomicType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) + union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) } } class Probe extends ManagerToClientChannel with HasCacheBlockAddress { - val p_type = UInt(width = co.probeTypeWidth) + val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t - def makeRelease( - client_xact_id: UInt, - meta: ClientMetadata = co.clientMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(false), - r_type = co.getReleaseType(this, meta), - client_xact_id = client_xact_id, - addr_block = this.addr_block, - addr_beat = addr_beat, - data = data) - } } object Probe { - val co = new Probe().co def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } - - def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { - apply(co.getProbeType(M_FLUSH, meta), addr_block) - } } @@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) + val r_type = UInt(width = tlCoh.releaseTypeWidth) val voluntary = Bool() // Helper funcs def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) //TODO: Assumes all releases write back full cache blocks: - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary - def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { - Grant( - g_type = Grant.voluntaryAck, - builtin_type = Bool(true), // Grant.voluntaryAck is built-in type - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id - ) - } + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } object Release { - val co = new Release().co def apply( voluntary: Bool, r_type: UInt, @@ -346,68 +324,52 @@ object Release { rel.voluntary := voluntary rel } - - def makeVoluntaryWriteback( - meta: ClientMetadata, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(true), - r_type = co.getReleaseType(M_FLUSH, meta), - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = addr_beat, - data = data) - } } class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val builtin_type = Bool() - val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) // Helper funcs - def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(builtin_type, + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), - co.grantTypesWithData.contains(g_type)) + tlCoh.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(builtin_type, + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), Grant.typesWithMultibeatData.contains(g_type), - co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) - def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = new Finish + val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) f.manager_xact_id := this.manager_xact_id f } } object Grant { - val nBuiltinGrantTypes = 5 - //TODO Use Enum - def voluntaryAck = UInt(0) - def uncachedRead = UInt(1) - def uncachedReadBlock = UInt(2) - def uncachedWrite = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) - def typesWithMultibeatData= Vec(uncachedReadBlock) + val nBuiltInTypes = 4 + def voluntaryAckType = UInt("b00") + def ackType = UInt("b01") + def dataBeatType = UInt("b10") + def dataBlockType = UInt("b11") + def typesWithData = Vec(dataBlockType, dataBeatType) + def typesWithMultibeatData= Vec(dataBlockType) def apply( - builtin_type: Bool, + is_builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.builtin_type := builtin_type + gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -415,23 +377,12 @@ object Grant { gnt.data := data gnt } - - def getGrantTypeForUncached(a: Acquire): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedWriteBlock -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class Finish extends ClientToManagerChannel with HasManagerTransactionId - // Complete IO definitions for two types of TileLink clients -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) @@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](utl: UncachedTileLinkIO) = { + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { val conv = Module(new TileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: TileLinkIO) = tl } abstract trait HasArbiterTypes { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 1230771c..3c334376 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,437 +2,147 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] case object NIncoherentClients extends Field[Int] case object NCoherentClients extends Field[Int] -case object L2StoreDataQueueDepth extends Field[Int] -case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] -case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object L2CoherencePolicy extends Field[CoherencePolicy] -abstract trait CoherenceAgentParameters extends UsesParameters - with TileLinkParameters { +trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nCoherentClients = params(NCoherentClients) val nIncoherentClients = params(NIncoherentClients) val nClients = nCoherentClients + nIncoherentClients - val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) - val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases + def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerDataBeats = outerTLParams(TLDataBeats) + val outerDataBits = outerTLParams(TLDataBits) + def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerDataBeats = innerTLParams(TLDataBeats) + val innerDataBits = innerTLParams(TLDataBits) + val innerBeatAddrBits = log2Up(innerDataBeats) + val innerByteAddrBits = log2Up(innerDataBits/8) + require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } +abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters +abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters -class DataQueueLocation extends Bundle with CoherenceAgentParameters { - val idx = UInt(width = dqIdxBits) - val loc = UInt(width = log2Ceil(nDataQueueLocations)) -} -object DataQueueLocation { - def apply(idx: UInt, loc: UInt) = { - val d = new DataQueueLocation - d.idx := idx - d.loc := loc - d +trait HasCoherenceAgentWiringHelpers { + def doOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( + out: DecoupledIO[S], + ins: Seq[DecoupledIO[S]]) { + def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter( + out.bits.clone, + ins.size, + out.bits.payload.tlDataBeats, + lock _)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } + + def doInputRouting[T <: HasManagerTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { + val idx = in.bits.payload.manager_xact_id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + in.ready := Vec(outs.map(_.ready)).read(idx) } } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } +trait HasInnerTLIO extends CoherenceAgentBundle { + val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val incoherent = Vec.fill(nClients){Bool()}.asInput + def iacq(dummy: Int = 0) = inner.acquire.bits.payload + def iprb(dummy: Int = 0) = inner.probe.bits.payload + def irel(dummy: Int = 0) = inner.release.bits.payload + def ignt(dummy: Int = 0) = inner.grant.bits.payload + def ifin(dummy: Int = 0) = inner.finish.bits.payload } -class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) { - - val internalDataBits = new DataQueueLocation().getWidth - val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - - // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Queue to store impending UncachedWrite data - val acquire = io.inner.acquire - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val) - val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } - - // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - - // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() - val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width - val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } - - // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Wire probe requests to clients - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() - val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - - // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) - val is_in_sdq = outer_data_ptr.loc === inStoreQueue - val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && - outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( - inStoreQueue -> sdq(outer_data_ptr.idx), - inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer <> outer_arb.io.out - - // Update SDQ valid bits - when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | - PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) - } +trait HasUncachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } - -abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) - val nClients = params(NCoherentClients) + params(NIncoherentClients) - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cfin = io.inner.finish.bits - val macq = io.outer.acquire.bits - val mgnt = io.outer.grant.bits - +trait HasCachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new TileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def oprb(dummy: Int = 0) = outer.probe.bits.payload + def orel(dummy: Int = 0) = outer.release.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) +class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_inner_data = Reg(init=Bool(false)) - // TODO: assert that all releases have full blocks of data - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) - - io.has_acquire_conflict := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - - when(collect_inner_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(inner_data_cnt) := crel.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_inner_data := crel.payload.hasMultibeatData() - state := Mux(crel.payload.hasData(), s_outer, - Mux(crel.payload.requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } +abstract class CoherenceAgent extends CoherenceAgentModule { + def innerTL: TileLinkIO + def outerTL: TileLinkIO + def incoherent: Vec[Bool] } -class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) +abstract class ManagerCoherenceAgent extends CoherenceAgent + with HasCoherenceAgentWiringHelpers { + val io = new ManagerTLIO + def innerTL = io.inner + def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def incoherent = io.incoherent +} - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) +class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO - val collect_inner_data = Reg(init=Bool(false)) - //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) +abstract class HierarchicalCoherenceAgent extends CoherenceAgent { + val io = new HierarchicalTLIO + def innerTL = io.inner + def outerTL = io.outer + def incoherent = io.incoherent +} +trait HasTrackerConflictIO extends Bundle { + val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) +} - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) +class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO +class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO - val pending_outer_write = xact.hasData() - val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - - io.has_acquire_conflict := xact.conflicts(cacq.payload) && - !collect_inner_data && - (state != s_idle) - io.has_release_match := xact.conflicts(crel.payload) && - !crel.payload.isVoluntary() && - (state != s_idle) - - val outer_write_acq = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = crel.payload.addr_beat, - data = crel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe() - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - when(collect_inner_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := cacq.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } +abstract class XactTracker extends CoherenceAgentModule { + def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = cacq.payload.hasData() - val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) - when(io.inner.acquire.valid) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_inner_data := cacq.payload.hasMultibeatData() - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } - } - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { - when(crel.payload.hasData()) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - when(outer_data_done) { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } .otherwise { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { // Read data from outer memory (possibly what was just written) - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { state := s_mem_resp } - } - is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - io.outer.acquire.bits.payload := outer_write_acq - when(outer_data_done) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_mem_resp) { // Wait to forward grants from outer memory - when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(cgnt_data_done) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { // Wait for transaction to complete - when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { - state := s_idle - } - } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + beat: UInt = UInt(0)) = { + connectDataBeatCounter(in.fire(), in.bits.payload, beat) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } } From 8e41fcf6fc58f5194f70fbbf6ed057c9972b7340 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 13:13:12 -0800 Subject: [PATCH 289/374] reduce MemIFTag size, enable non pow2 HellaFLowQueue size --- uncore/src/main/scala/memserdes.scala | 68 ++++++++++++--------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index f10b3257..7542815b 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -244,20 +244,13 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { (io.tl.release.fire() && rel_has_data), tlDataBeats) val tl_done_out = Reg(init=Bool(false)) val make_grant_ack = Reg(init=Bool(false)) - val grant_for_rel = Grant( - is_builtin_type = Bool(true), - g_type = Grant.voluntaryAckType, - client_xact_id = tag_out, - manager_xact_id = UInt(0)) - val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_out(0), - a_type = tag_out >> UInt(1), - client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) + gnt_arb.io.in(1).bits.payload := Grant( + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -278,7 +271,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := io.tl.release.bits.payload.client_xact_id + tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) @@ -289,8 +283,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data data_from_rel := Bool(false) @@ -358,7 +351,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -366,8 +360,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_cmd.bits.rw := acq_has_data io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data } @@ -409,12 +402,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_in(0), - a_type = tag_in >> UInt(1), - client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) @@ -442,23 +435,22 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := - ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = io.mem.resp.bits.tag(0), - a_type = io.mem.resp.bits.tag >> UInt(1), - client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) + require(entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow @@ -466,7 +458,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 + val (deq_ptr, deq_done) = Counter(do_deq, entries) when (do_enq != do_deq) { maybe_full := do_enq } val ptr_match = enq_ptr === deq_ptr @@ -482,7 +474,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + ram_addr := Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) From df79e7ff8dcadc09d91d6b2dede2b583487f2af0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 14:40:31 -0800 Subject: [PATCH 290/374] secondary miss bug --- uncore/src/main/scala/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2524f3d1..1e8d1b60 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -60,9 +60,7 @@ trait HasClientSideCoherencePolicy { first_cmd: UInt, second_cmd: UInt, meta: ClientMetadata): Bool = { - isWriteIntent(second_cmd) && - !isWriteIntent(first_cmd) || - (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + isWriteIntent(second_cmd) && !isWriteIntent(first_cmd) } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: From 002f1a1b393df17b4fe0e07f0a43dc36bdc0ac4f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 12:40:37 -0700 Subject: [PATCH 291/374] pin outer finish header --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75eb897d..aa2097af 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -831,6 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.ognt().requiresAck()) { pending_finish.payload := pending_finish_on_ognt pending_finish.header.dst := io.outer.grant.bits.header.src + pending_finish.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, From a1f04386f75664654a049c1d5b4078f402515c70 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 16:34:59 -0700 Subject: [PATCH 292/374] Headerless TileLinkIO and arbiters --- uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/tilelink.scala | 123 ++++++++++++++++++++------- 2 files changed, 92 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4d5e1135..644a8317 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new TileLinkIO + val mem = new HeaderlessUncachedTileLinkIO val scr = new SCRIO } @@ -195,19 +195,16 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq - io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + io.mem.acquire.bits := Mux(cmd === cmd_writemem, PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - io.mem.probe.ready := Bool(false) - io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c8996ddf..3d46ae80 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import scala.math.max +import scala.reflect.ClassTag // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -290,7 +291,6 @@ object Probe { } } - class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -420,6 +420,42 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO) = tl } +// This version of TileLinkIO does not contain network headers for packets +// that originate in the Clients (i.e. Acquire and Release). These headers +// are provided in the top-level that instantiates the clients and network. +// By eliding the header subbundles within the clients we can enable +// hierarchical P&R while minimizing unconnected port errors in GDS. +class HeaderlessUncachedTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new Acquire) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new Release) +} + +class HeaderlessTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new HeaderlessUncachedTileLinkIO().flip + val out = new HeaderlessTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} + +object HeaderlessTileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId @@ -428,6 +464,7 @@ abstract trait HasArbiterTypes { HasClientTransactionId with HasTileLinkData } + // Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule with HasArbiterTypes { @@ -438,34 +475,65 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) - out <> arb.io.out - ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) req.ready := arb.ready }} + arb.io.out <> mngr } - def hookupManagerSource[M <: ManagerSourcedWithId] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { - out.ready := Bool(false) + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + arb.io.out <> mngr + } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) for (i <- 0 until arbN) { - ins(i).valid := Bool(false) - when (arbIdx(out.bits.payload) === UInt(i)) { - ins(i).valid := out.valid - out.ready := ins(i).ready + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready } - ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := + managerSourcedClientXactId(mngr.bits.payload) } } + + def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_||_) + } } abstract class UncachedTileLinkIOArbiter(n: Int) @@ -474,13 +542,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { @@ -488,18 +552,11 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - io.in.map{ _.probe.valid := io.out.probe.valid } - io.in.map{ _.probe.bits := io.out.probe.bits } - io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } // Appends the port index of the arbiter to the client_xact_id From 17072a00416470e93e0b9ffc817f89b950753c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Mar 2015 01:15:03 -0700 Subject: [PATCH 293/374] L2 Writeback bugfix --- uncore/src/main/scala/cache.scala | 27 +++++++++++-------- uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 37 +++++++++++++++++---------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index aa2097af..ec25501c 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -566,17 +566,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && - xact_meta.coh.outer.requiresVoluntaryWriteback() - val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + (xact_meta.coh.outer.requiresVoluntaryWriteback() || + xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( + val pending_icoh_on_irel = pending_coh.inner.onRelease( incoming = io.irel(), - src = io.inner.release.bits.header.src), - pending_coh.outer.onHit(M_XWR)) // WB is a write + src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, pending_coh.outer.onGrant(io.ognt(), xact.op_code())) @@ -768,7 +767,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) val _needs_writeback = !_tag_match && do_allocate && - _coh.outer.requiresVoluntaryWriteback() + (_coh.outer.requiresVoluntaryWriteback() || + _coh.inner.requiresProbesOnVoluntaryWriteback()) val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { @@ -796,12 +796,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - pending_coh := pending_coh_on_irel + pending_coh.inner := pending_icoh_on_irel // Handle released dirty data //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { irel_had_data := Bool(true) + pending_coh.outer := pending_ocoh_on_irel mergeDataInner(data_buffer, io.irel()) } // We don't decrement release_count until we've received all the data beats. @@ -931,8 +932,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val resp_data_done = connectInternalDataBeatCounter(io.data.resp) val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), + incoming = io.irel(), src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) @@ -1008,6 +1010,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle released dirty data when(io.irel().hasData()) { irel_had_data := Bool(true) + xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. @@ -1016,7 +1019,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(irel_had_data, s_outer_release, s_data_read) + state := Mux(irel_had_data, // If someone released a dirty block + s_outer_release, // write that block back, otherwise + Mux(xact_coh.outer.requiresVoluntaryWriteback(), + s_data_read, // write extant dirty data back, or just + s_wb_resp)) // drop a clean block after collecting acks } } is(s_data_read) { diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1e8d1b60..862eb484 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -383,6 +383,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), @@ -501,6 +502,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7542815b..eafc44bd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -212,6 +212,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) val tlDataBits = params(TLDataBits) @@ -235,8 +236,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(Bits()) - val addr_out = Reg(Bits()) + val tag_out = Reg(UInt(width = mifTagBits)) + val addr_out = Reg(UInt(width = mifAddrBits)) val has_data = Reg(init=Bool(false)) val data_from_rel = Reg(init=Bool(false)) val (tl_cnt_out, tl_wrap_out) = @@ -343,26 +344,34 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid io.mem.req_cmd.valid := Bool(true) cmd_sent_out := io.mem.req_cmd.ready - tag_out := io.mem.req_cmd.bits.tag - addr_out := io.mem.req_data.bits.data - has_data := io.mem.req_cmd.bits.rw tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) make_grant_ack := Bool(true) - io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data + val tag = Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) + val addr = io.tl.release.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := rel_has_data + tag_out := tag + addr_out := addr + has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data - io.mem.req_cmd.bits.rw := acq_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_cmd.bits.rw := acq_has_data + val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.isBuiltInType()) + val addr = io.tl.acquire.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := acq_has_data + tag_out := tag + addr_out := addr + has_data := acq_has_data } } } @@ -380,7 +389,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } when(tl_wrap_out) { tl_done_out := Bool(true) } when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) + gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } } when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { From 3ab1aca7decddfa7e1bf5ee35c864c43076e23ba Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 11 Mar 2015 01:56:47 -0700 Subject: [PATCH 294/374] L2 subblock access bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ec25501c..1d117ded 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -412,7 +412,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { - val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } From b4ed1d912160964c5ea41a1b2614afa21c83f71f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 14:28:17 -0700 Subject: [PATCH 295/374] Add builtin prefetch types to TileLink --- uncore/src/main/scala/tilelink.scala | 48 ++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3d46ae80..ef0d3307 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -109,6 +109,8 @@ class Acquire extends ClientToManagerChannel def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) + // Assumes no custom types have data def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) @@ -124,7 +126,8 @@ class Acquire extends ClientToManagerChannel Acquire.getBlockType -> Grant.dataBlockType, Acquire.putType -> Grant.ackType, Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType)) + Acquire.putAtomicType -> Grant.dataBeatType, + Acquire.prefetchType -> Grant.ackType)) } } @@ -136,6 +139,7 @@ object Acquire { def putType = UInt("b010") def putBlockType = UInt("b011") def putAtomicType = UInt("b100") + def prefetchType = UInt("b101") def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -172,9 +176,9 @@ object Acquire { // Asks for a single TileLink beat of data object Get { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -189,8 +193,8 @@ object Get { // Asks for an entire cache block of data object GetBlock { def apply( - client_xact_id: UInt = UInt(0), - addr_block: UInt, + client_xact_id: UInt = UInt(0), + addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -201,6 +205,22 @@ object GetBlock { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with read permissions +object GetPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XRD, Bool(true))) + } +} + // Writes up to a single TileLink beat of data, using mask object Put { def apply( @@ -275,6 +295,22 @@ object PutAtomic { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with write permissions +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) From 059575c334dd34cc818231fdd5ff82769171c28f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 15:43:41 -0700 Subject: [PATCH 296/374] cleanup mergeData and prep for cleaner data_buffer in L2 --- uncore/src/main/scala/cache.scala | 72 +++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 2 + 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1d117ded..87473587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -6,15 +6,16 @@ import Chisel._ case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] -case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object CacheBlockBytes extends Field[Int] +case object CacheBlockOffsetBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) - val blockOffBits = params(BlockOffBits) + val blockOffBits = params(CacheBlockOffsetBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = params(PAddrBits) - untagBits @@ -165,6 +166,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val blockAddrBits = params(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats + val internalDataBeats = params(CacheBlockBytes)*8/rowBits require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) @@ -410,6 +412,14 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + class CacheBlockBuffer { + val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) + + def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) + def inner = Vec.fill(innerDataBeats){ Bits(width = innerDataBits) }.fromBits(buffer) + def outer = Vec.fill(outerDataBeats){ Bits(width = outerDataBits) }.fromBits(buffer) + } + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) @@ -554,9 +564,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) - } + } + val amo_result = Reg(io.iacq().data.clone) val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -612,44 +623,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer(0) // default + amoalu.io.rhs := data_buffer.head // default - // TODO: figure out how to merge the following three versions of this func - def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) + def mergeData[T <: HasTileLinkData] + (byteAddrBits: Int, dataBits: Int) + (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + val old_data = incoming + val new_data = buffer(beat) val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = + val valid_beat = xact.addr_beat === beat + val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + UInt(0, width = dataBits))) + buffer(beat) := ~wmask & old_data | wmask & Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result - } - def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) - def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } + val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ + val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ + val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === @@ -694,7 +692,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { manager_xact_id = UInt(trackerId), addr_beat = ignt_data_cnt, data = Mux(xact.is(Acquire.putAtomicType), - data_buffer(innerDataBeats), + amo_result, data_buffer(ignt_data_cnt))) io.inner.acquire.ready := Bool(false) @@ -803,7 +801,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel()) + mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -824,7 +822,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt()) + mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -851,13 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3c334376..05319ee9 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -21,6 +21,8 @@ trait CoherenceAgentParameters extends UsesParameters { def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerByteAddrBits = log2Up(outerDataBits/8) def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) From 1aff919c24a80b54e95efa5f1389da1df7b4f1aa Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 17:32:06 -0700 Subject: [PATCH 297/374] added prefetchAck Grant type --- uncore/src/main/scala/memserdes.scala | 6 +++--- uncore/src/main/scala/tilelink.scala | 29 ++++++++++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eafc44bd..b9786b4c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -249,7 +249,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(1).valid := Bool(false) gnt_arb.io.in(1).bits.payload := Grant( is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) @@ -414,7 +414,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = tag_in(0), g_type = Mux(tag_in(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), @@ -447,7 +447,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = io.mem.resp.bits.tag(0), g_type = Mux(io.mem.resp.bits.tag(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = io.mem.resp.bits.tag >> UInt(1), manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ef0d3307..531d3f52 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -121,13 +121,13 @@ class Acquire extends ClientToManagerChannel def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType def getBuiltInGrantType(dummy: Int = 0): UInt = { - MuxLookup(this.a_type, Grant.ackType, Array( - Acquire.getType -> Grant.dataBeatType, - Acquire.getBlockType -> Grant.dataBlockType, - Acquire.putType -> Grant.ackType, - Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType, - Acquire.prefetchType -> Grant.ackType)) + MuxLookup(this.a_type, Grant.putAckType, Array( + Acquire.getType -> Grant.getDataBeatType, + Acquire.getBlockType -> Grant.getDataBlockType, + Acquire.putType -> Grant.putAckType, + Acquire.putBlockType -> Grant.putAckType, + Acquire.putAtomicType -> Grant.getDataBeatType, + Acquire.prefetchType -> Grant.prefetchAckType)) } } @@ -389,13 +389,14 @@ class Grant extends ManagerToClientChannel } object Grant { - val nBuiltInTypes = 4 - def voluntaryAckType = UInt("b00") - def ackType = UInt("b01") - def dataBeatType = UInt("b10") - def dataBlockType = UInt("b11") - def typesWithData = Vec(dataBlockType, dataBeatType) - def typesWithMultibeatData= Vec(dataBlockType) + val nBuiltInTypes = 5 + def voluntaryAckType = UInt("b000") + def putAckType = UInt("b001") + def prefetchAckType = UInt("b011") + def getDataBeatType = UInt("b100") + def getDataBlockType = UInt("b101") + def typesWithData = Vec(getDataBlockType, getDataBeatType) + def typesWithMultibeatData= Vec(getDataBlockType) def apply( is_builtin_type: Bool, From f75126c39c9925fc2073a82e03c270372e6bb06d Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 298/374] Require self probes for all built-in Acquire types This ensures that puts by the RoCC accelerator properly invalidates its tile's L1 D$, with which it currently shares the same TileLink port. --- uncore/src/main/scala/tilelink.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 531d3f52..7ea79ad1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -117,8 +117,9 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache + // and nbdcache invalidations after RoCC accelerator puts + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 8f8022379c11755367c0f35fce6a4775be91e06b Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 299/374] Fix AMO opcode extraction --- uncore/src/main/scala/tilelink.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7ea79ad1..d353205e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -96,7 +96,9 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_code(dummy: Int = 0) = Mux(isBuiltInType() && + (a_type === Acquire.putType || a_type === Acquire.putBlockType), + M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) From 5e40c8ba77c7067fa686434db61cd9da4e3bb84f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 14:36:46 -0700 Subject: [PATCH 300/374] write back meta data when cache miss even when coherence meta data is clean --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 87473587..9815855e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val meta = pending_coh_on_ignt != xact_meta.coh - when(meta) { pending_coh := pending_coh_on_ignt } - state := Mux(meta, s_meta_write, + val coh_dirty = pending_coh_on_ignt != xact_meta.coh + when(coh_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(!is_hit || coh_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From 2c31ed64261cafefb0273cb41dde5d002052dc73 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 15:34:20 -0700 Subject: [PATCH 301/374] previous bug fix for meta data writeback wasn't quite right --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9815855e..7e601ba0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val coh_dirty = pending_coh_on_ignt != xact_meta.coh - when(coh_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(!is_hit || coh_dirty, s_meta_write, + val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + when(meta_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From dcc84c4dd3e429235a4739ebc3edaf18f891f149 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 09:33:56 -0700 Subject: [PATCH 302/374] arbiter probe ready bugfix --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d353205e..7da4cfab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -572,7 +572,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr: DecoupledIO[LogicalNetworkIO[M]]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } - mngr.ready := clts.map(_.ready).reduce(_||_) + mngr.ready := clts.map(_.ready).reduce(_&&_) } } From 8181262419de27535ce1fb396e82739533217fad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 16:22:14 -0700 Subject: [PATCH 303/374] clean up incoherent and probe flags --- uncore/src/main/scala/broadcast.scala | 30 ++++++++++++++------------- uncore/src/main/scala/cache.scala | 29 +++++++++++++------------- uncore/src/main/scala/uncore.scala | 2 +- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index bd8a3f24..e28cbf1c 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -219,16 +219,15 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence + val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), - UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.incoherent.toBits | myflag) + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -307,18 +306,21 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, + val needs_probes = mask_incoherent.orR + when(needs_probes) { + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) + } + state := Mux(needs_probes, s_probe, Mux(pending_outer_write_, s_mem_write, Mux(pending_outer_read_, s_mem_read, s_make_grant))) } } is(s_probe) { // Generate probes - io.inner.probe.valid := probe_flags.orR + io.inner.probe.valid := pending_probes.orR when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data to be written back diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e601ba0..dfe5b10a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -596,13 +596,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) @@ -770,8 +771,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(_tag_match, Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade @@ -920,9 +921,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_finish = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.wb.req.bits.coh.inner.full() + val mask_incoherent = full_sharers & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) @@ -985,12 +988,10 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val coh = io.wb.req.bits.coh - val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(needs_probes, s_probe, s_data_read) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 05319ee9..0d414f0d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -73,7 +73,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nClients){Bool()}.asInput + val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload From 3a78ca210d27e194c47ff3216539de5ef6f248c5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 16:33:35 -0700 Subject: [PATCH 304/374] bugfix in uncached TL to TL convertors --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7da4cfab..1aac8ce4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -443,7 +443,7 @@ class TileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } object TileLinkIOWrapper { @@ -484,7 +484,7 @@ class HeaderlessTileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } From 6e540825b214ac5c5a53a1f1e19fbaa213853145 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 14 Mar 2015 02:15:24 -0700 Subject: [PATCH 305/374] Use entire 12-bit CSR address --- uncore/src/main/scala/htif.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 644a8317..81d95453 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -34,7 +34,7 @@ class HostIO extends HTIFBundle class PCRReq extends Bundle { val rw = Bool() - val addr = Bits(width = 5) + val addr = Bits(width = 12) val data = Bits(width = 64) } @@ -206,7 +206,6 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) @@ -214,7 +213,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -234,7 +233,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === pcr_reset) { + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From c03976896e8908fc306e108d83e5dd759ae2a5db Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 16:53:15 -0700 Subject: [PATCH 306/374] separate queues for resp tag and data --- uncore/src/main/scala/memserdes.scala | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b9786b4c..0a2b9c39 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -33,9 +33,9 @@ class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag - +class MemTag extends HasMemTag class MemData extends HasMemData +class MemResp extends HasMemData with HasMemTag class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) @@ -543,9 +543,20 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq + // Have separate queues to allow for different mem implementations + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) + resp_dataq.io.enq.valid := io.mem.resp.valid + resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + + val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tagq.io.enq.valid := io.mem.resp.valid + resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid + io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag + resp_dataq.io.deq.ready := io.cpu.resp.ready + resp_tagq.io.deq.ready := io.cpu.resp.ready inc := resp_dataq.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw From 23a6b007c1f6f357abfe2bad1209f8241bb31c9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 23:10:51 -0700 Subject: [PATCH 307/374] Fix BroadcastHub AcquiteTracker allocation bug and clean up tracker wiring --- uncore/src/main/scala/broadcast.scala | 100 +++++++++++++++----------- uncore/src/main/scala/cache.scala | 45 +++++++----- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index e28cbf1c..c8014f45 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -40,53 +40,53 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerList.map(_.io.incoherent := io.incoherent.toBits) // Queue to store impending Put data - val acquire = io.inner.acquire + val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) } val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData() + when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.bits.payload.data := + DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + sdq_rdy && !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val voluntary = io.irel().isVoluntary() + val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -107,7 +107,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val free_sdq = io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -216,8 +216,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && - Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), - "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType, + Acquire.prefetchType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) @@ -238,17 +239,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ognt_ack = Reg(init=Bool(false)) val pending_outer_write = xact.hasData() - val pending_outer_read = io.ignt().hasData() val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read = io.ignt().hasData() val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() io.has_acquire_conflict := xact.conflicts(io.iacq()) && - !collect_iacq_data && - (state != s_idle) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state != s_idle) - io.has_acquire_match := Bool(false) + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_probe) val outer_write_acq = Bundle(PutBlock( client_xact_id = UInt(trackerId), @@ -282,6 +284,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dfe5b10a..ef3cb3d7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -356,33 +356,34 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.inner.acquire val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val acquireList = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => - arb.valid := acq.ready - acq.bits := acquire.bits - acq.valid := arb.ready && (acquire_idx === UInt(i)) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires // Wire releases from clients - val release = io.inner.release val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).indexWhere{b: Bool => b} - val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - releaseList.zipWithIndex.map { case(r, i) => - r.bits := release.bits - r.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(releaseList.map(_.ready)).read(release_idx) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -658,8 +659,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data - io.has_release_match := !io.irel().isVoluntary() && - (xact.addr_block === io.irel().addr_block) && + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && (state === s_probe) // If we're allocating in this cache, we can use the current metadata @@ -729,6 +730,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { From f6d1a2fb76cfe73ff87149264e9ac2f1237a82bb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:38 -0700 Subject: [PATCH 308/374] No more self-probes required --- uncore/src/main/scala/tilelink.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1aac8ce4..7bd218ad 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -119,9 +119,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache - // and nbdcache invalidations after RoCC accelerator puts - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From b72230a9f04f18d9358ca427b69445ff090e6d95 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:55 -0700 Subject: [PATCH 309/374] PutBlock bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ef3cb3d7..8c65a20f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -636,7 +636,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.addr_beat === beat + val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), From f35a6a574f3386d282f9fcd9ef1fd692fe9e9c1c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:25:01 -0700 Subject: [PATCH 310/374] Add a queue on released data coming in to L2 --- uncore/src/main/scala/cache.scala | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8c65a20f..bc12cfeb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -322,14 +322,22 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nWays)) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray(1)) - tshrfile.io.inner <> io.inner - tshrfile.io.meta <> meta.io - tshrfile.io.data <> data.io + //TODO: Expose queue depths and data array pipeline cycles as parameters? + tshrfile.io.inner.acquire <> io.inner.acquire + tshrfile.io.inner.probe <> io.inner.probe + tshrfile.io.inner.release <> Queue(io.inner.release) + tshrfile.io.inner.grant <> io.inner.grant + tshrfile.io.inner.finish <> io.inner.finish + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent + + val meta = Module(new L2MetadataArray) + tshrfile.io.meta <> meta.io + + val data = Module(new L2DataArray(1)) + tshrfile.io.data <> data.io } class TSHRFileIO extends HierarchicalTLIO { From cf1df2d72a3b3cd687eed7f7ed8c3a17ac62758e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 311/374] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 145e15701e8e2dc6b34c252487201c0801b0d8cb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 312/374] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 36fc67dc7c1011afa763350c819657a911fdd74b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 313/374] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..b6d22d5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -831,7 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -852,7 +852,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -879,13 +879,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From f6fe037e303d708101c0d1d7b5716a9efb3919ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 314/374] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 260 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 114 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b6d22d5e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,38 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -819,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -829,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -839,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -853,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -864,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -873,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -915,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -952,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -988,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1029,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 40a5059cee6311f6bd5e74fc4a46aee5f34d99fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 315/374] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From 672fbb574f8d34066f946680d5c34addf6d2614e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 316/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From b21fcdfbe0836646a2a5a13f0a5b1369dab6c7e1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 317/374] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 2d33dd8e6ee2dbf257a046a2619f89c75b613fee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 318/374] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From fd8f6ff0269808bdc9bc2dc24a28d22e7f5890d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 319/374] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 257dcedcb34b341197983eba6df4f64786d0e2ae Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 320/374] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From ce9d4b6e70192c0353056c59dda435c2e58ab740 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 321/374] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 0e51fef200776fd411b9c92e3d5eca4dc0c4d81b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 322/374] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 1471d9debc04a81c98c3af32d1846bfde77cce7a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 05:40:05 -0700 Subject: [PATCH 323/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..469adc94 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -754,6 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") From 9bedde9a8a7317e601bf920a013ccea01b08f85e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 12:22:57 -0700 Subject: [PATCH 324/374] re-merge mem resp queues --- uncore/src/main/scala/memserdes.scala | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 0a2b9c39..a5bea3a3 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,21 +544,11 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) - resp_dataq.io.enq.valid := io.mem.resp.valid - resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_data_q.io.enq <> io.mem.resp + io.cpu.resp <> resp_data_q.io.deq - val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) - resp_tagq.io.enq.valid := io.mem.resp.valid - resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag - - io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid - io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data - io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag - resp_dataq.io.deq.ready := io.cpu.resp.ready - resp_tagq.io.deq.ready := io.cpu.resp.ready - - inc := resp_dataq.io.deq.fire() + inc := resp_data_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 6af48e168a7e2ab8de76b67477c3619aad034b41 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 325/374] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 469adc94..08ce9c3e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -832,7 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -853,7 +853,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -880,13 +880,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From 3f070eee1f849ff3c273c77ce732f650587241df Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 326/374] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 261 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08ce9c3e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,39 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -820,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -830,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -840,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -854,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -865,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -874,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -916,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -953,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -989,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1030,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 6d565d22e3e991ecb6abdfa75b1d19053024dbb3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 327/374] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From d9598d26f250ec6863ca8d1c82956287848b1e9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 328/374] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From 23f8033df55c59a85422b4213ed2bebc24f72437 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 329/374] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 730a13abf27d9fc15902b23970097f90860acfd7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 330/374] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From dc8861409449c6e1560da3d6389ddc1d71cfbdef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 331/374] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 5c2461c74339da1698fcb08d539c95d8be5b122b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 332/374] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From fc0ae81a977b096bd2b94d6b2b031f206341a13f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 15:04:30 -0700 Subject: [PATCH 333/374] added finish counter --- uncore/src/main/scala/cache.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c52566c3..d4dbd75f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -610,12 +610,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) + when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -789,8 +789,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) - irel_had_data := Bool(false) - ognt_had_data := Bool(false) + ifin_cnt := UInt(0) state := s_meta_read } } @@ -940,7 +939,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_inner_finish) { io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - UInt(1) + when(ifin_cnt <= UInt(1)) { state := s_idle } + } } } From a52a729ab90bac736e6f5235518b475be26bb18a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 15:54:21 -0700 Subject: [PATCH 334/374] bugfix wmask handling --- uncore/src/main/scala/cache.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d4dbd75f..ed8d7051 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -658,13 +658,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) + val wmask = FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) - when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } + when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ val mergeDataInner = mergeData(innerDataBits) _ From 4fd01d82b86fffdca3364e1c47546311d66ef8e8 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 17:48:02 -0700 Subject: [PATCH 335/374] don't block finish messages when grant'ing --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed8d7051..cd68a8df 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -929,6 +929,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) + } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -942,6 +946,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } + when(ifin_cnt === UInt(0)) { state := s_idle } } } From b08dced37cbe61589355c8505a4403544d35f311 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 17:51:00 -0700 Subject: [PATCH 336/374] first cut at pending scoreboarding --- uncore/src/main/scala/cache.scala | 90 +++++++++++++++++------------- uncore/src/main/scala/uncore.scala | 7 +++ 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index cd68a8df..0680442d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -446,17 +446,28 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( in: DecoupledIO[T], beat: UInt = UInt(0), full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.fire(), in.bits, beat, full_block) } + def connectInternalDataBeatCounter[T <: HasL2Data]( in: ValidIO[T], full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } + + def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + + def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -591,8 +602,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_finish = Reg{ io.outer.finish.bits.clone } - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -611,18 +620,38 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mask_incoherent = mask_self & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ofin = Reg{ io.outer.finish.bits.clone } + + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + ignt_q.io.enq.valid := Bool(false) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + ignt_q.io.deq.ready := ignt_data_done + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_reads := pending_reads | + addPendingBit(io.inner.acquire) | + dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) + + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + pending_writes := pending_writes | + addPendingBit(io.inner.acquire) | + addPendingBit(io.inner.release) | + addPendingBit(io.outer.grant) & + dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_resps := pending_resps | + addInternalPendingBit(io.data.read) & + dropInternalPendingBit(io.data.resp) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -720,8 +749,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) io.outer.finish.valid := Bool(false) - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -739,7 +768,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amo_result, data_buffer(ignt_data_idx))) io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id - ignt_q.io.deq.ready := ignt_data_done io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || @@ -789,6 +817,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_writes := UInt(0) pending_resps := UInt(0) ifin_cnt := UInt(0) + ignt_q.io.enq.valid := Bool(true) state := s_meta_read } } @@ -842,7 +871,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) - pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -866,14 +894,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src - pending_finish.header.src := UInt(bankId) + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src + pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, @@ -892,18 +919,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := pending_reads.orR when(io.data.read.ready) { - pending_resps := pending_resps | UIntToOH(curr_read_beat) - pending_reads := pending_reads & ~UIntToOH(curr_read_beat) when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - } - when(io.data.read.ready && io.data.resp.valid) { - pending_resps := (pending_resps & - ~UIntToOH(io.data.resp.bits.addr_beat)) | - UIntToOH(curr_read_beat) } } is(s_data_resp) { @@ -918,7 +937,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_write) { io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present when(io.data.write.ready) { - pending_writes := pending_writes & ~UIntToOH(curr_write_beat) when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } @@ -950,20 +968,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } - ignt_q.io.enq.valid := io.inner.acquire.fire() && - (state === s_idle || !xact.hasMultibeatData()) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat // Handle Get and Put merging - when(io.inner.acquire.fire()) { - val beat = io.iacq().addr_beat - when(io.iacq().hasData()) { - mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - //iacq_data_valid(beat) := Bool(true) - pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) - } - when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } + when(io.inner.acquire.fire() && io.iacq().hasData()) { + mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && @@ -1007,7 +1017,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_finish = Reg{ io.outer.finish.bits.clone } + val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -1043,8 +1053,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1133,8 +1143,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src state := s_outer_finish }.otherwise { state := s_wb_resp diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0d414f0d..16e976ad 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -139,12 +139,19 @@ abstract class XactTracker extends CoherenceAgentModule { val done = Mux(multi, multi_done, inc) (cnt, done) } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], beat: UInt = UInt(0)) = { connectDataBeatCounter(in.fire(), in.bits.payload, beat) } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } + + def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat)) + } } From 504eedbf893404b3ed3ee1caf7dcec82161d492f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:07:52 -0700 Subject: [PATCH 337/374] fixes in in bit manipulation --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0680442d..e3783d93 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -464,10 +464,10 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = - Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = - Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -634,22 +634,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := pending_reads | - addPendingBit(io.inner.acquire) | + pending_reads := (pending_reads | + addPendingBit(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := pending_writes | + pending_writes := (pending_writes | addPendingBit(io.inner.acquire) | addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant) & + addPendingBit(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := pending_resps | - addInternalPendingBit(io.data.read) & + pending_resps := (pending_resps | + addInternalPendingBit(io.data.read)) & dropInternalPendingBit(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( From 2d7375760dd6e5c89ef019a6809833ff1157f39a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:35:04 -0700 Subject: [PATCH 338/374] set pending_writes for puts --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3783d93..27c2e64f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -814,7 +814,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux(io.iacq().isSubBlockType(), UIntToOH(io.iacq().addr_beat), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := UInt(0) + pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) From d14efce0b4a6eaea52bc6d83f6cd89839a547e53 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 19:54:11 -0700 Subject: [PATCH 339/374] fix wmask_buffer initialization --- uncore/src/main/scala/cache.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 27c2e64f..9dc35500 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -676,12 +676,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) - wmask_buffer(beat) := wmask | wmask_buffer(beat) - } - def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data val new_data = data_buffer(beat) // Newly Put data is already in the buffer @@ -971,7 +965,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { - mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } //iacq_data_valid(beat) := Bool(true) } From 9de5161d7ab8d351a3e86cbed7d8ad7faaac7b25 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 20:24:04 -0700 Subject: [PATCH 340/374] guard all writes to data ram with masks --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9dc35500..7e14c8f4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -686,6 +686,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) + wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ @@ -779,7 +780,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) From 2d3f947a9c7247317aa9db5b4be30b9e2abe6e86 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:33:05 -0700 Subject: [PATCH 341/374] cleaned up finish counter --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e14c8f4..c7d857d3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -632,7 +632,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } + when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } + .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBit(io.inner.acquire)) & @@ -943,12 +945,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) - } } is(s_meta_write) { io.meta.write.valid := Bool(true) + io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -956,14 +956,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_finish) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } when(ifin_cnt === UInt(0)) { state := s_idle } } } - // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { val beat = io.iacq().addr_beat From b9591b297c94106de1b5bf4560a68a7b95e7751b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:59:22 -0700 Subject: [PATCH 342/374] added s_wait_puts to L2AcquireTracker --- uncore/src/main/scala/cache.scala | 33 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c7d857d3..70411a14 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -591,7 +591,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -602,12 +602,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_more_put_data = !pending_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -808,9 +811,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), - SInt(-1, width = innerDataBeats)).toUInt + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) @@ -876,7 +882,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } when(release_count === UInt(0)) { state := Mux(is_hit, - Mux(pending_writes.orR, s_data_write, s_data_read), + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read), s_outer_acquire) } } @@ -901,7 +909,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } } @@ -910,7 +920,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } is(s_data_read) { @@ -927,12 +939,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) } } } + is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } From 638bace858b78f7a8bae7bccf6398ead12203d5f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:28:06 -0700 Subject: [PATCH 343/374] avoid reading data when write mask is full --- uncore/src/main/scala/cache.scala | 19 ++++++++++++------- uncore/src/main/scala/uncore.scala | 7 ++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 70411a14..b193fc8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -603,7 +603,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) + pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -640,15 +640,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBit(io.inner.acquire)) & + addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) pending_writes := (pending_writes | - addPendingBit(io.inner.acquire) | - addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant)) & + addPendingBitWhenHasData(io.inner.acquire) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) @@ -815,9 +815,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { UInt(0), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := addPendingBit(io.inner.acquire) + pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) @@ -933,6 +933,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } + when(PopCount(pending_reads) === UInt(0)) { + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) + } } is(s_data_resp) { when(io.data.resp.valid) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 16e976ad..14414b76 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -150,8 +150,13 @@ abstract class XactTracker extends CoherenceAgentModule { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } - def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & UIntToOH(in.bits.payload.addr_beat)) } + + def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat)) + } } From d48775eecb3c7bc36b10959e9fdda6c8562483a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:31:23 -0700 Subject: [PATCH 344/374] cleanup outdated comments --- uncore/src/main/scala/cache.scala | 9 --------- 1 file changed, 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193fc8a..8bcf3885 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -869,8 +869,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) @@ -990,18 +988,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle From 0e4cf74d8ab6c3fedebc0afae80404bf9978267b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:53:27 -0700 Subject: [PATCH 345/374] always merge Puts --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8bcf3885..455f85ef 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -173,7 +173,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 val enableGetMerging = false - val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -709,11 +708,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { do_allocate && ignt_q.io.enq.ready //TODO: mix Puts and PutBlocks - val can_merge_iacq_put = ((Bool(enablePutMerging) && - (xact.isBuiltInType(Acquire.putType) && - io.iacq().isBuiltInType(Acquire.putType))) || - (xact.isBuiltInType(Acquire.putBlockType) && - io.iacq().isBuiltInType(Acquire.putBlockType))) && + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType)) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && (xact_src === io.inner.acquire.bits.header.src) && (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && From f4f59464df9bce2c8ccec3a9b7a9a698fa53ae7f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 21:44:20 -0700 Subject: [PATCH 346/374] fix pending_puts initialization --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..514878e4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -810,7 +810,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), + addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), addPendingBitWhenWmaskIsNotFull(io.inner.acquire), From aa5435800d7e7f757752817229fc8fd956d1c1f3 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 22:43:00 -0700 Subject: [PATCH 347/374] fix get merging, and always turn it on --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 514878e4..27bb898b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -172,7 +172,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 - val enableGetMerging = false } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -697,8 +696,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mergeDataInner = mergeData(innerDataBits) _ val mergeDataOuter = mergeData(outerDataBits) _ - val can_merge_iacq_get = Bool(enableGetMerging) && - (xact.isBuiltInType(Acquire.getType) && + val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && @@ -979,13 +977,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } // Handle Get and Put merging - when(io.inner.acquire.fire() && io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + when(io.inner.acquire.fire()) { + when (io.iacq().hasData()) { + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) + } + when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } } assert(!(state != s_idle && io.inner.acquire.fire() && From 825c4b28509a81ad1d17e0df80c56d5384130f80 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:44:53 -0700 Subject: [PATCH 348/374] make ignts more eager --- uncore/src/main/scala/cache.scala | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..39b5e7c2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,13 +459,16 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } - def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + def addPendingBitInternal[T <: HasL2BeatAddr](in: DecoupledIO[T]) = Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + def addPendingBitInternal[T <: HasL2BeatAddr](in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & UIntToOH(in.bits.addr_beat) + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) - def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } @@ -626,6 +629,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ofin = Reg{ io.outer.finish.bits.clone } + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + pending_ignt_data := pending_ignt_data | + addPendingBitInternal(io.data.resp) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) @@ -634,8 +642,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } - .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + when(ignt_data_done) { + ifin_cnt := Mux(io.inner.finish.fire(), + Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), + Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) + } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | @@ -653,8 +664,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) pending_resps := (pending_resps | - addInternalPendingBit(io.data.read)) & - dropInternalPendingBit(io.data.resp) + addPendingBitInternal(io.data.read)) & + dropPendingBitInternal(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, @@ -755,7 +766,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid + io.inner.grant.valid := state === s_inner_grant && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( @@ -770,8 +784,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) + io.inner.release.ready := state === s_probe + io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, + s_data_write, s_wait_puts, s_data_resp).contains(state) io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) @@ -817,6 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) + pending_ignt_data := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) state := s_meta_read @@ -863,7 +879,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data @@ -954,27 +969,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_grant) { - when(ignt_data_done && ignt_q.io.count === UInt(1)) { + when(ignt_q.io.count === UInt(0) || + (ignt_data_done && ignt_q.io.count === UInt(1))) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, - Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), + s_inner_finish, s_idle)) } - io.inner.finish.ready := Bool(true) } is(s_meta_write) { io.meta.write.valid := Bool(true) - io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - when(ifin_cnt <= UInt(1)) { state := s_idle } + when(ifin_cnt === UInt(0) || + (io.inner.finish.valid && ifin_cnt === UInt(1))) { + state := s_idle } - when(ifin_cnt === UInt(0)) { state := s_idle } } } From 42aa4aa8cab91fad1a5220108f16a47e63483f61 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:53:50 -0700 Subject: [PATCH 349/374] Secondary miss param --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b632bbaa..98e122d4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -10,6 +10,8 @@ case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object NPrimaryMisses extends Field[Int] +case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] @@ -171,7 +173,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states - val nSecondaryMisses = 4 + val nSecondaryMisses = params(NSecondaryMisses) } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters From e325399c8781181b9d030d7bb4f8d85c87d0ea54 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 12:49:53 -0700 Subject: [PATCH 350/374] Re-split mem resp tag and data queues --- uncore/src/main/scala/memserdes.scala | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index a5bea3a3..e2d6df85 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,11 +544,21 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_data_q.io.enq <> io.mem.resp - io.cpu.resp <> resp_data_q.io.deq + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemData }) + resp_data_q.io.enq.valid := io.mem.resp.valid + resp_data_q.io.enq.bits.data := io.mem.resp.bits.data - inc := resp_data_q.io.deq.fire() + val resp_tag_q = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tag_q.io.enq.valid := io.mem.resp.valid + resp_tag_q.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_data_q.io.deq.valid && resp_tag_q.io.deq.valid + io.cpu.resp.bits.data := resp_data_q.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tag_q.io.deq.bits.tag + resp_data_q.io.deq.ready := io.cpu.resp.ready + resp_tag_q.io.deq.ready := io.cpu.resp.ready + + inc := resp_data_q.io.deq.fire() && resp_tag_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 1ff184bf62ea41370a0ee083ee69257dae3dd86f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 17:55:05 -0700 Subject: [PATCH 351/374] first cut at optimized state transitions --- uncore/src/main/scala/cache.scala | 169 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 19 +++- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 98e122d4..85fedbbc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -174,6 +174,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) + val isLastLevelCache = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -594,7 +595,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -605,15 +606,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) + val present_puts = Reg(init=Bits(0, width = innerDataBeats)) + present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val is_hit = xact_tag_match && + (xact_meta.coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + xact_meta.coh.outer.isValid())) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) - val needs_more_put_data = !pending_puts.andR + val needs_more_put_data = !present_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -652,7 +656,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & - dropPendingBit(io.data.read) + (dropPendingBit(io.data.read) & + dropPendingBitWhenWmaskIsFull(io.inner.acquire) & + dropPendingBitWhenHasData(io.inner.release) & + dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -683,6 +690,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), pending_coh.outer) + val pending_ofin_on_ognt = io.ognt().makeFinish() val amo_result = xact.data val amoalu = Module(new AMOALU) @@ -714,11 +722,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish).contains(state) && do_allocate && ignt_q.io.enq.ready - //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && @@ -727,7 +735,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish, s_data_read, s_data_resp).contains(state) && do_allocate && @@ -737,7 +745,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().addr_block(idxMSB,idxLSB) io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && - (state === s_probe) + (state === s_inner_probe) io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && (state != s_idle) && @@ -746,7 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := Bool(false) + io.outer.acquire.valid := state === s_outer_acquire io.outer.acquire.bits.payload := Mux(do_allocate, xact_meta.coh.outer.makeAcquire( client_xact_id = UInt(trackerId), @@ -756,12 +764,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.bits.header.src := UInt(bankId) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) + io.outer.grant.ready := state === s_outer_grant + io.outer.finish.valid := state === s_outer_finish io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() - io.inner.probe.valid := Bool(false) + io.inner.probe.valid := state === s_inner_probe && pending_probes.orR io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) @@ -784,34 +791,34 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_probe + io.inner.release.ready := state === s_inner_probe io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := Bool(false) + io.data.read.valid := state === s_data_read && pending_reads.orR io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := Bool(false) + io.data.write.valid := state === s_data_write && pending_writes.orR io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := Bool(false) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := Bool(false) + io.wb.req.valid := state === s_wb_req io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en @@ -824,12 +831,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + Mux(io.iacq().isSubBlockType(), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt) pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) @@ -838,10 +847,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_meta_read } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } + is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match @@ -850,31 +856,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _is_hit = _tag_match && + (_coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + _coh.outer.isValid())) + val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_probes) { + when(_needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + state := Mux(!_tag_match, + Mux(_needs_writeback, s_wb_req, s_outer_acquire), + Mux(_needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } - } - is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_acquire } - } - is(s_probe) { + is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } + is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_inner_probe) { // Send probes - io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } @@ -892,21 +900,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read), - s_outer_acquire) - } - } - is(s_outer_acquire) { - io.outer.acquire.valid := Bool(true) - when(oacq_data_done) { - state := s_outer_grant + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) } } + is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } is(s_outer_grant) { - io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) @@ -919,51 +920,40 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } } } is(s_outer_finish) { - io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } is(s_data_read) { - io.data.read.valid := pending_reads.orR - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } - } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } - when(PopCount(pending_reads) === UInt(0)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) + when(io.data.read.ready) { + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) - } + } + when(PopCount(pending_resps) === UInt(0) || + (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } @@ -978,15 +968,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { s_inner_finish, s_idle)) } } - is(s_meta_write) { - io.meta.write.valid := Bool(true) + is(s_meta_write) { when(io.meta.write.ready) { state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + (io.inner.finish.valid && ifin_cnt === UInt(1))) { state := s_idle } } @@ -1031,7 +1020,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1057,12 +1046,13 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { incoming = io.irel(), src = io.inner.release.bits.header.src) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write + val pending_ofin_on_ognt = io.ognt().makeFinish() io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) io.has_release_match := !io.irel().isVoluntary() && io.irel().conflicts(xact_addr_block) && - (state === s_probe) + (state === s_inner_probe) io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) @@ -1076,7 +1066,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1109,15 +1098,15 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_probes) { + val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(needs_probes, s_probe, s_data_read) + state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } } - is(s_probe) { + is(s_inner_probe) { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 14414b76..cfd33567 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -151,12 +151,23 @@ abstract class XactTracker extends CoherenceAgentModule { } def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat) } + def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | + ~UIntToOH(in.bits.payload.addr_beat) + } + + //TODO | with existing wmask_buffer? def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat) + } + + def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | + ~UIntToOH(in.bits.payload.addr_beat) } } From 19059bf0ebc0e3e2f2c6bfb746ff19a04208e6dd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:28:03 -0700 Subject: [PATCH 352/374] put data can be used for ignts --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 85fedbbc..7257a699 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -638,7 +638,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ignt_data := pending_ignt_data | addPendingBitInternal(io.data.resp) | addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) + addPendingBitWhenHasData(io.outer.grant) | + addPendingBitWhenHasData(io.inner.acquire) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) From fb8071c12d90b20803c933f7a5a416855691cc44 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:49:32 -0700 Subject: [PATCH 353/374] generous hit detection on PutBlocks --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7257a699..741e168d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -609,10 +609,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && - (xact_meta.coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - xact_meta.coh.outer.isValid())) + val is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || @@ -857,10 +857,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && - (_coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - _coh.outer.isValid())) + val _is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + _coh.outer.isValid()) + else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || From b92ea608914be20519f57b3d1189c27cd794c85c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 19:32:46 -0700 Subject: [PATCH 354/374] you can 'hit' with putblocks even when the tag doesn't match but you still better writeback --- uncore/src/main/scala/cache.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 741e168d..146250bc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -859,7 +859,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = (if(isLastLevelCache) (xact.isBuiltInType(Acquire.putBlockType) || - _coh.outer.isValid()) + _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && @@ -871,17 +871,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(!_tag_match, - Mux(_needs_writeback, s_wb_req, s_outer_acquire), + state := Mux(_needs_writeback, s_wb_req, Mux(_needs_inner_probes, s_inner_probe, - Mux(!is_hit, s_outer_acquire, + Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_wb_resp) { + when(io.wb.resp.valid) { + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + } + } is(s_inner_probe) { // Send probes when(io.inner.probe.ready) { From 002851f836b350b34e90aa2fb6eda44b5b18af7b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 21:11:40 -0700 Subject: [PATCH 355/374] disentangle is_hit logic --- uncore/src/main/scala/cache.scala | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 146250bc..55fc544d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,16 +608,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - - val is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) - val do_allocate = xact.allocate() - val needs_writeback = !xact_tag_match && do_allocate && - (xact_meta.coh.outer.requiresVoluntaryWriteback() || - xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_more_put_data = !present_puts.andR + val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -861,7 +853,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.isBuiltInType(Acquire.putBlockType) || _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) @@ -881,11 +872,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { - state := Mux(!is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + when(io.wb.resp.valid) { + val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!_skip_outer_acquire, s_outer_acquire, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -907,7 +897,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(!is_hit, s_outer_acquire, + val _skip_outer_acquire = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else xact_meta.coh.outer.isHit(xact.op_code())) + state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))) @@ -968,7 +962,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { when(ignt_q.io.count === UInt(0) || (ignt_data_done && ignt_q.io.count === UInt(1))) { - val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), From 004ad11af6fa5636ac046871284347a156a72444 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:14:41 -0700 Subject: [PATCH 356/374] cleanup pending signals --- uncore/src/main/scala/cache.scala | 23 ++++++++++------------- uncore/src/main/scala/uncore.scala | 14 +++++--------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 55fc544d..ed10e3c5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -606,9 +606,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val present_puts = Reg(init=Bits(0, width = innerDataBeats)) - present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = !present_puts.andR + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) + val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -648,9 +648,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & + addPendingBitWhenGetOrAtomic(io.inner.acquire)) & (dropPendingBit(io.data.read) & - dropPendingBitWhenWmaskIsFull(io.inner.acquire) & dropPendingBitWhenHasData(io.inner.release) & dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) @@ -824,14 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - addPendingBitWhenHasData(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), - Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt) + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux(io.iacq().isBuiltInType(), + addPendingBitWhenGetOrAtomic(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cfd33567..de55c0e2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -160,14 +160,10 @@ abstract class XactTracker extends CoherenceAgentModule { ~UIntToOH(in.bits.payload.addr_beat) } - //TODO | with existing wmask_buffer? - def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | - ~UIntToOH(in.bits.payload.addr_beat) + def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + val a = in.bits.payload + Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && + (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & + UIntToOH(a.addr_beat) } } From 3cf033180f2980a34b07016a4197d949e57a9b5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:41:09 -0700 Subject: [PATCH 357/374] pending read fix --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed10e3c5..5f656f31 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -826,9 +826,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(), - addPendingBitWhenGetOrAtomic(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || + !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) From 4176edaa34956cc1ce08d108d790aebf267bdd1f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 10:17:51 -0700 Subject: [PATCH 358/374] clean up tracker allocation --- uncore/src/main/scala/broadcast.scala | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 2564140b..64a96cf3 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,28 +48,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + val block_acquires = acquireConflicts.orR || !sdq_rdy + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && + !block_acquires && + (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -80,17 +76,21 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles From b7af610569ce10c952519359f63cbb10509c2b43 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 11:29:04 -0700 Subject: [PATCH 359/374] broadcast hub bugfix --- uncore/src/main/scala/broadcast.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 64a96cf3..ce72f625 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -229,9 +229,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) From 8959b2e81a0ae6fb198a37a72ebc25d19af4b1ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 13:29:52 -0700 Subject: [PATCH 360/374] TileLinkEnqueuer --- uncore/src/main/scala/tilelink.scala | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 42c17ca4..0aff9aab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -505,6 +505,29 @@ object HeaderlessTileLinkIOWrapper { } } +class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) + io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) + io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) + io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) + io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths))(p) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { + apply(in, (depth, depth, depth, depth, depth))(p) + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId From c941f0a68ec202f96bade0bcf1a48ec8151785d5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 27 Mar 2015 16:21:29 -0700 Subject: [PATCH 361/374] New virtual memory implementation (Sv39) --- uncore/src/main/scala/memserdes.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e2d6df85..059fbdcf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -7,8 +7,9 @@ import scala.math._ case object PAddrBits extends Field[Int] case object VAddrBits extends Field[Int] case object PgIdxBits extends Field[Int] +case object PgLevels extends Field[Int] +case object PgLevelBits extends Field[Int] case object ASIdBits extends Field[Int] -case object PermBits extends Field[Int] case object PPNBits extends Field[Int] case object VPNBits extends Field[Int] From ced627f00a94ac772fc3bfa8e19ddecfae203769 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 19 Mar 2015 01:11:34 -0700 Subject: [PATCH 362/374] slight mod to pending_puts cleaner state transition logic --- uncore/src/main/scala/cache.scala | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 5f656f31..a705f618 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,7 +608,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -816,6 +815,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + when(io.data.resp.valid) { + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + } + + def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) + def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) switch (state) { is(s_idle) { when(io.inner.acquire.valid) { @@ -865,7 +870,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } @@ -873,7 +878,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.wb.resp.valid) { val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + Mux(pending_puts.orR, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -902,7 +907,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))) } } is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } @@ -921,7 +926,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { }.otherwise { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } } @@ -930,36 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.finish.ready) { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } is(s_data_read) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + when(doneNextCycleHot(io.data.read.ready, pending_reads)) { + state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(PopCount(pending_resps) === UInt(0) || - (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(pending_puts.orR, s_wait_puts, s_data_write)) + } + } + is(s_wait_puts) { + when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { + state := s_data_write } } - is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } is(s_inner_grant) { - when(ignt_q.io.count === UInt(0) || - (ignt_data_done && ignt_q.io.count === UInt(1))) { + when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -973,8 +974,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_finish) { - when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { state := s_idle } } From 9708d25dfff7b4a0a6381be398f00dad608a9436 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Mar 2015 02:06:53 -0700 Subject: [PATCH 363/374] Restructure L2 state machine and utilize HeaderlessTileLinkIO --- uncore/src/main/scala/broadcast.scala | 43 +- uncore/src/main/scala/cache.scala | 709 ++++++++++++-------------- uncore/src/main/scala/htif.scala | 44 +- uncore/src/main/scala/memserdes.scala | 28 +- uncore/src/main/scala/metadata.scala | 4 +- uncore/src/main/scala/network.scala | 5 +- uncore/src/main/scala/tilelink.scala | 393 +++++++++++--- uncore/src/main/scala/uncore.scala | 93 ++-- uncore/src/main/scala/util.scala | 26 +- 9 files changed, 780 insertions(+), 565 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index ce72f625..98c8f0af 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -52,20 +52,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits - val acquire_idx = Mux(acquireMatches.orR, + val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) val block_acquires = acquireConflicts.orR || !sdq_rdy - io.inner.acquire.ready := acquireReadys.orR && !block_acquires + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := - DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && - !block_acquires && - (acquire_idx === UInt(i)) + tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -94,24 +91,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + io.inner.grant.bits.payload.data := io.outer.grant.bits.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } // Get the pending data out of the store data queue - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && + io.outer.acquire.bits.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( + io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -147,7 +144,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.outer.acquire.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -159,11 +155,12 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? - io.outer.acquire.bits.payload := Bundle(PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)))(outerTLParams) + io.outer.acquire.bits := Bundle( + PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) when(collect_irel_data) { io.inner.release.ready := Bool(true) @@ -271,7 +268,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact addr_block = xact.addr_block))(outerTLParams) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default + io.outer.acquire.bits := outer_read //default io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -346,7 +343,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact when(io.inner.release.valid) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel + io.outer.acquire.bits := outer_write_rel when(io.outer.acquire.ready) { when(oacq_data_done) { pending_ognt_ack := Bool(true) @@ -368,7 +365,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) - io.outer.acquire.bits.payload := outer_write_acq + io.outer.acquire.bits := outer_write_acq when(oacq_data_done) { pending_ognt_ack := Bool(true) state := Mux(pending_outer_read, s_mem_read, s_mem_resp) @@ -376,7 +373,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := !pending_ognt_ack - io.outer.acquire.bits.payload := outer_read + io.outer.acquire.bits := outer_read when(io.outer.acquire.fire()) { state := s_mem_resp } } is(s_mem_resp) { // Wait to forward grants from outer memory diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a705f618..05588203 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -329,22 +330,13 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nSets)) require(isPow2(nWays)) + val meta = Module(new L2MetadataArray) // TODO: add delay knob + val data = Module(new L2DataArray(1)) val tshrfile = Module(new TSHRFile(bankId)) - - //TODO: Expose queue depths and data array pipeline cycles as parameters? - tshrfile.io.inner.acquire <> io.inner.acquire - tshrfile.io.inner.probe <> io.inner.probe - tshrfile.io.inner.release <> Queue(io.inner.release) - tshrfile.io.inner.grant <> io.inner.grant - tshrfile.io.inner.finish <> io.inner.finish - + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent - - val meta = Module(new L2MetadataArray) tshrfile.io.meta <> meta.io - - val data = Module(new L2DataArray(1)) tshrfile.io.data <> data.io } @@ -364,6 +356,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule Module(new L2AcquireTracker(id, bankId)) } + // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) @@ -373,21 +366,15 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + val block_acquires = acquireConflicts.orR + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits @@ -395,15 +382,18 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule } // Wire releases from clients - val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ - wb.io.has_release_match).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - trackerReleaseIOs.zipWithIndex.foreach { + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) + trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.release.bits tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -412,7 +402,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) + val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -483,10 +473,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } + val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val coh = xact_meta.coh + val coh = xact_old_meta.coh val collect_irel_data = Reg(init=Bool(false)) val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -501,7 +491,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -529,8 +518,8 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) - io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) when(collect_irel_data) { @@ -561,7 +550,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta + xact_old_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(xact.hasData(), s_data_write, s_meta_write), @@ -595,101 +584,73 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) + // State holding transaction metadata val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } - val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val do_allocate = xact.allocate() + val xact_old_meta = Reg{ new L2Metadata } + val pending_coh = Reg{ xact_old_meta.coh.clone } - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val pending_ofin = Reg{ io.outer.finish.bits.clone } - - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) - pending_ignt_data := pending_ignt_data | - addPendingBitInternal(io.data.resp) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) | - addPendingBitWhenHasData(io.inner.acquire) + // Secondary miss queue val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - ignt_q.io.enq.valid := Bool(false) ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - ignt_q.io.deq.ready := ignt_data_done - - val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { - ifin_cnt := Mux(io.inner.finish.fire(), - Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), - Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) - } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + // TODO add ignt.dst <- iacq.src + // State holding progress made on processing this transaction + val iacq_data_done = + connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = + connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = + connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = + connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := (pending_reads | - addPendingBitWhenGetOrAtomic(io.inner.acquire)) & - (dropPendingBit(io.data.read) & - dropPendingBitWhenHasData(io.inner.release) & - dropPendingBitWhenHasData(io.outer.grant)) - val curr_read_beat = PriorityEncoder(pending_reads) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := (pending_writes | - addPendingBitWhenHasData(io.inner.acquire) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant)) & - dropPendingBit(io.data.write) - val curr_write_beat = PriorityEncoder(pending_writes) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := (pending_resps | - addPendingBitInternal(io.data.read)) & - dropPendingBitInternal(io.data.resp) + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_meta_write = Reg{ Bool() } - val pending_coh_on_hit = HierarchicalMetadata( - io.meta.resp.bits.meta.coh.inner, - io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_icoh_on_irel = pending_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) - val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write - val pending_coh_on_ognt = HierarchicalMetadata( - ManagerMetadata.onReset, - pending_coh.outer.onGrant(io.ognt(), xact.op_code())) - val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), - pending_coh.outer) - val pending_ofin_on_ognt = io.ognt().makeFinish() + val all_pending_done = + !(pending_reads.orR || + pending_writes.orR || + pending_resps.orR || + pending_puts.orR || + pending_ognts || + ignt_q.io.count > UInt(0) || + //pending_meta_write || // Has own state: s_meta_write + pending_ifins) - val amo_result = xact.data + // Provide a single ALU per tracker to merge Puts and AMOs with data being + // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer.head // default + amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData + val amo_result = xact.data // Reuse xact buffer space to store AMO result + + // Utility functions for updating the data and metadata that will be kept in + // the cache or granted to the original requestor after this transaction: + + def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { + when(flag && pending_coh != next) { + pending_meta_write := Bool(true) + pending_coh := next + } + } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data @@ -704,104 +665,182 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(rowBits) _ - val mergeDataInner = mergeData(innerDataBits) _ - val mergeDataOuter = mergeData(outerDataBits) _ + def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } + } + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { + when(in.fire() && in.bits.payload.hasData()) { + mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + } + } + def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + // Actual transaction processing logic begins here: + // + // First, take care of accpeting new requires or secondary misses + // For now, the only allowed secondary miss types are Gets-under-Get + // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - (xact_src === io.inner.acquire.bits.header.src) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish).contains(state) && - do_allocate && - ignt_q.io.enq.ready + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready + // This logic also allows the tail beats of a PutBlock to be merged in val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - (xact_src === io.inner.acquire.bits.header.src) && - (xact.client_xact_id === io.iacq().client_xact_id) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish, s_data_read, - s_data_resp).contains(state) && - do_allocate && - ignt_q.io.enq.ready - - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) - io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !io.has_acquire_match - - // If we're allocating in this cache, we can use the current metadata - // to make an appropriate custom Acquire, otherwise we copy over the - // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := state === s_outer_acquire - io.outer.acquire.bits.payload := Mux(do_allocate, - xact_meta.coh.outer.makeAcquire( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - op_code = xact.op_code()), - Bundle(Acquire(xact))(outerTLParams)) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := state === s_outer_grant - io.outer.finish.valid := state === s_outer_finish - io.outer.finish.bits := pending_ofin - - io.inner.probe.valid := state === s_inner_probe && pending_probes.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - - io.inner.grant.valid := state === s_inner_grant && - ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_inner_probe - io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, - s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := state === s_data_read && pending_reads.orR + // Track whether any beats are missing from a PutBlock + pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_probe_dst + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + io.inner.release.ready := state === s_inner_probe + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( // Drop sharer + incoming = io.irel(), + src = io.inner.release.bits.header.src), + Mux(io.irel().hasData(), // Dirty writeback + pending_coh.outer.onHit(M_XWR), + pending_coh.outer)) + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + mergeDataInner(io.inner.release) + + // The following subtrascation handles misses or coherence permission + // upgrades by initiating a new transaction in the outer memory: + // + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := state === s_outer_acquire + io.outer.acquire.bits := Mux( + xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( + client_xact_id = UInt(0), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + + // Probes from the outer memory system are handled in the WritebackUnit + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + + io.outer.grant.ready := state === s_busy + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.outer.grant.bits, xact.op_code())) + updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) + mergeDataOuter(io.outer.grant) + + // Going back to the original inner transaction, we can issue a Grant as + // soon as the data is released, granted, put, or read from the cache + pending_ignt_data := pending_ignt_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) | + addPendingBitInternal(io.data.resp) + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.deq.ready := ignt_data_done + // Make the Grant message using the data stored in the secondary miss queue + io.inner.grant.valid := state === s_busy && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) + // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of + // multiple client transaction ids from merged secondary misses + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) + + // We must wait for as many Finishes as we sent Grants + io.inner.finish.ready := state === s_busy + + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)) | + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && + pending_reads.orR && + !pending_ognts io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := state === s_data_write && pending_writes.orR + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + mergeDataInternal(io.data.resp) + + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && + pending_writes.orR && + !pending_ognts && + !pending_reads(curr_write_beat) && + !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -810,188 +849,106 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + // Handling of secondary misses (Gets and Puts only for now) + when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) } - def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) - def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) - switch (state) { - is(s_idle) { - when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src - xact := io.iacq() - xact.data := UInt(0) - wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - dropPendingBitWhenHasData(io.inner.acquire), - UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || - !io.iacq().isBuiltInType(), - SInt(-1, width = innerDataBeats), - addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt - pending_writes := addPendingBitWhenHasData(io.inner.acquire) - pending_resps := UInt(0) - pending_ignt_data := UInt(0) - ifin_cnt := UInt(0) - ignt_q.io.enq.valid := Bool(true) - state := s_meta_read - } - } - is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - pending_coh := io.meta.resp.bits.meta.coh - val _coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - _tag_match && _coh.outer.isValid()) - else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && - (_coh.outer.requiresVoluntaryWriteback() || - _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) - when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_inner_probes) { - pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) - } - state := Mux(_needs_writeback, s_wb_req, - Mux(_needs_inner_probes, s_inner_probe, - Mux(!_is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) - } - } - is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { - when(io.wb.resp.valid) { - val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_inner_probe) { - // Send probes - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - when(io.inner.release.valid) { - pending_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(io.irel().addr_beat, io.irel().data) - } - // We don't decrement release_count until we've received all the data beats. - when(!io.irel().hasMultibeatData() || irel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - val _skip_outer_acquire = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else xact_meta.coh.outer.isHit(xact.op_code())) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))) - } - } - is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } - is(s_outer_grant) { - when(io.outer.grant.valid) { - when(io.ognt().hasData()) { - mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - } - when(ognt_data_done) { - pending_coh := pending_coh_on_ognt - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - pending_ofin.header.src := UInt(bankId) - state := s_outer_finish - }.otherwise { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - } - } - is(s_outer_finish) { - when(io.outer.finish.ready) { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - is(s_data_read) { - when(doneNextCycleHot(io.data.read.ready, pending_reads)) { - state := s_data_resp - } - } - is(s_data_resp) { - when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_wait_puts) { - when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { - state := s_data_write - } - } - is(s_data_write) { - when(io.data.write.ready) { - when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } - } - } - is(s_inner_grant) { - when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { - val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh - when(meta_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(meta_dirty, s_meta_write, - Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), - s_inner_finish, s_idle)) - } - } - is(s_meta_write) { - when(io.meta.write.ready) { - state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { - state := s_idle - } - } + // Defined here because of Chisel default wire demands, used in s_meta_resp + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + xact.data := UInt(0) + pending_puts := Mux( + io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenBeatHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux( + io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) + pending_resps := UInt(0) + pending_ignt_data := UInt(0) + pending_meta_write := UInt(0) + state := s_meta_read + } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + val coh = io.meta.resp.bits.meta.coh + val tag_match = io.meta.resp.bits.tag_match + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || + tag_match && coh.outer.isValid()) + val needs_writeback = !tag_match && + xact.allocate() && + (coh.outer.requiresVoluntaryWriteback() || + coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) + when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } + pending_coh := Mux(is_hit, pending_coh_on_hit, coh) + when(needs_inner_probes) { + val full_sharers = coh.inner.full() + val mask_self = Mux( + xact.requiresSelfProbe(), + coh.inner.full() | UIntToOH(xact_src), + coh.inner.full() & ~UIntToOH(xact_src)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + pending_iprbs := mask_incoherent + } + state := Mux(needs_writeback, s_wb_req, + Mux(needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, s_busy))) + } + when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } + when(state === s_wb_resp && io.wb.resp.valid) { + // If we're overwriting the whole block in a last level cache we can + // just do it without fetching any data from memory + val skip_outer_acquire = Bool(isLastLevelCache) && + xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + // Tag matches, so if this is the last level cache we can use the data without upgrading permissions + val skip_outer_acquire = + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_outer_acquire && oacq_data_done) { state := s_busy } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && (io.meta.write.ready || !pending_meta_write)) { + wmask_buffer.foreach { w => w := UInt(0) } + state := s_idle } - // Handle Get and Put merging - when(io.inner.acquire.fire()) { - when (io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - } - when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - } + // These IOs are used for routing in the parent + val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_inner_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match + //TODO: relax from in_same_set to xact.conflicts(io.iacq())? + // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") @@ -1019,7 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1027,10 +984,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() @@ -1056,15 +1012,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) // default - io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.release.bits.header.src := UInt(bankId) + io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) io.outer.grant.ready := Bool(false) // default - io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_ofin io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1100,7 +1053,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_inner_probes) { pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) + irel_cnt := PopCount(mask_incoherent) } state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } @@ -1121,12 +1074,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } - // We don't decrement release_count until we've received all the data beats. + // We don't decrement irel_cnt until we've received all the data beats. when(!io.irel().hasData() || irel_data_done) { - release_count := release_count - UInt(1) + irel_cnt := irel_cnt - UInt(1) } } - when(release_count === UInt(0)) { + when(irel_cnt === UInt(0)) { state := Mux(irel_had_data, // If someone released a dirty block s_outer_release, // write that block back, otherwise Mux(xact_coh.outer.requiresVoluntaryWriteback(), @@ -1152,19 +1105,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_outer_grant) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - state := s_outer_finish - }.otherwise { - state := s_wb_resp - } + state := s_wb_resp } } - is(s_outer_finish) { - io.outer.finish.valid := Bool(true) - when(io.outer.finish.ready) { state := s_wb_resp } - } is(s_wb_resp) { io.wb.resp.valid := Bool(true) state := s_idle diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 81d95453..787847b9 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -127,21 +127,11 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) - val mem_acked = Reg(init=Bool(false)) - val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt()) - val mem_needs_ack = Reg(Bool()) - when (io.mem.grant.valid) { - mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.manager_xact_id - mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := io.mem.grant.bits.payload.requiresAck() - } - io.mem.grant.ready := Bool(true) - - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) val state = Reg(init=state_rx) + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, @@ -149,28 +139,18 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, state_tx))) } - - val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || - (state === state_mem_rresp && io.mem.grant.valid), dataBeats) when (state === state_mem_wreq) { when (cnt_done) { state := state_mem_wresp } } when (state === state_mem_rreq) { when(io.mem.acquire.ready) { state := state_mem_rresp } } - when (state === state_mem_wresp) { - when (mem_acked) { - state := state_mem_finish - mem_acked := Bool(false) - } + when (state === state_mem_wresp && io.mem.grant.valid) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << offsetBits-3) } - when (state === state_mem_rresp) { - when (cnt_done) { - state := state_mem_finish - mem_acked := Bool(false) - } - } - when (state === state_mem_finish && io.mem.finish.ready) { + when (state === state_mem_rresp && cnt_done) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -187,8 +167,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_req_data = (0 until n).map { i => val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := - io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := + io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) } packet_ram(Cat(cnt, ui)) }.reverse.reduce(_##_) @@ -202,9 +182,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.manager_xact_id := mem_gxid - io.mem.finish.bits.header.dst := mem_gsrc + io.mem.grant.ready := Bool(true) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 059fbdcf..878ce19a 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -207,7 +207,7 @@ class MemDesser(w: Int) extends Module // test rig side //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val io = new Bundle { val tl = new TileLinkIO().flip val mem = new MemIO @@ -216,11 +216,10 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) - val tlDataBits = params(TLDataBits) - val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats - require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - require(params(TLClientXactIdBits) <= params(MIFTagBits)) + val dstIdBits = params(LNHeaderBits) + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") + require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) io.tl.acquire.ready := Bool(false) io.tl.probe.valid := Bool(false) @@ -231,6 +230,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) io.tl.grant <> gnt_arb.io.out + val dst_off = dstIdBits + tlClientXactIdBits val acq_has_data = io.tl.acquire.bits.payload.hasData() val rel_has_data = io.tl.release.bits.payload.hasData() @@ -253,6 +253,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) + gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(1).bits.header.src := UInt(0) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -273,7 +275,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data @@ -284,7 +287,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data @@ -350,7 +354,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.payload.client_xact_id, + val tag = Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) val addr = io.tl.release.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -364,7 +369,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + val tag = Cat(io.tl.acquire.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) val addr = io.tl.acquire.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -421,6 +427,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -454,6 +462,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) } } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 8f5d7e90..af5504c8 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -18,6 +18,7 @@ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + def !=(rhs: ClientMetadata): Bool = !this.===(rhs) def isValid(dummy: Int = 0): Bool = co.isValid(this) def isHit(cmd: UInt): Bool = co.isHit(cmd, this) @@ -99,6 +100,7 @@ class ManagerMetadata extends CoherenceMetadata { def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers + def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) def full(dummy: Int = 0) = co.dir.full(this.sharers) def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) @@ -171,7 +173,7 @@ class HierarchicalMetadata extends CoherenceMetadata { val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer - def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) + def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } object HierarchicalMetadata { diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 161a9a0c..7fdc8e09 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -6,6 +6,7 @@ import Chisel._ case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] class PhysicalHeader(n: Int) extends Bundle { val src = UInt(width = log2Up(n)) @@ -47,8 +48,8 @@ class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Opti abstract class LogicalNetwork extends Module class LogicalHeader extends Bundle { - val src = UInt(width = log2Up(params(LNEndpoints))) - val dst = UInt(width = log2Up(params(LNEndpoints))) + val src = UInt(width = params(LNHeaderBits)) + val dst = UInt(width = params(LNHeaderBits)) } class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 0aff9aab..c2bc2d30 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,7 +3,8 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -11,17 +12,22 @@ import scala.reflect.ClassTag case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] case object TLBlockAddrBits extends Field[Int] -case object TLManagerXactIdBits extends Field[Int] -case object TLClientXactIdBits extends Field[Int] +case object TLMaxClientXacts extends Field[Int] +case object TLMaxClientPorts extends Field[Int] +case object TLMaxManagerXacts extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { val tlBlockAddrBits = params(TLBlockAddrBits) - val tlClientXactIdBits = params(TLClientXactIdBits) - val tlManagerXactIdBits = params(TLManagerXactIdBits) + val tlMaxClientXacts = params(TLMaxClientXacts) + val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxManagerXacts = params(TLMaxManagerXacts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlDataBits = params(TLDataBits) + val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 @@ -41,12 +47,16 @@ abstract trait TileLinkParameters extends UsesParameters { val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters { +} abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel // Used to hook up logical network ports to physical network ports -trait TileLinkChannel extends TLBundle +trait TileLinkChannel extends TLBundle { + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool +} trait ClientToManagerChannel extends TileLinkChannel trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now @@ -97,8 +107,7 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType() && - (a_type === Acquire.putType || a_type === Acquire.putBlockType), + def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) @@ -328,6 +337,8 @@ class Probe extends ManagerToClientChannel val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) } object Probe { @@ -383,6 +394,7 @@ class Grant extends ManagerToClientChannel // Helper funcs def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t def is(t: UInt):Bool = g_type === t def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), @@ -394,7 +406,7 @@ class Grant extends ManagerToClientChannel def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) + val f = Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts }) f.manager_xact_id := this.manager_xact_id f } @@ -428,7 +440,10 @@ object Grant { } } -class Finish extends ClientToManagerChannel with HasManagerTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId { + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} // Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends TLBundle { @@ -455,33 +470,22 @@ class TileLinkIOWrapper extends TLModule { io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object TileLinkIOWrapper { - def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper)(p) - conv.io.in <> utl - conv.io.out - } - def apply(utl: UncachedTileLinkIO): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> utl - conv.io.out - } - def apply(tl: TileLinkIO) = tl -} // This version of TileLinkIO does not contain network headers for packets // that originate in the Clients (i.e. Acquire and Release). These headers // are provided in the top-level that instantiates the clients and network. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. +// Secondly, this version of the interface elides Finish messages, with the +// assumption that a FinishUnit has been coupled to the TileLinkIO port +// to deal with acking received Grants. class HeaderlessUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) + val grant = new DecoupledIO(new Grant).flip } class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } @@ -492,17 +496,231 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant - io.out.finish <> io.in.finish io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object HeaderlessTileLinkIOWrapper { +object TileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { val conv = Module(new HeaderlessTileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasClientId = TileLinkChannel with HasClientTransactionId + type LNAcquire = LogicalNetworkIO[Acquire] + type LNRelease = LogicalNetworkIO[Release] + type LNGrant = LogicalNetworkIO[Grant] + + def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( + in: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, beat) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, beat) + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} + } + } + + def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int): Vec[Bool] = { + val id = in.bits.payload.client_xact_id + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, UInt(0))._2 + case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} + } + } + + def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) + } + + def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[LogicalNetworkIO[T]], + down: DecoupledIO[LogicalNetworkIO[S]], + inc: T => Bool = (t: T) => Bool(true), + dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) + val do_inc = up_done && inc(up.bits.payload) + val do_dec = down_done && dec(down.bits.payload) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} + +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) +} + +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) + +class FinishUnit(srcId: Int = 0) extends TLModule + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + val grant_done = Bool(OUTPUT) + val pending_finish = Bool(OUTPUT) + } + + val entries = 1 << tlClientXactIdBits + val g = io.grant.bits.payload + assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) + + val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) + val q = Module(new FinishQueue(entries)) + + q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := io.grant.valid + io.refill.bits := io.grant.bits.payload + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) + + io.ready := q.io.enq.ready + io.grant_done := done + io.pending_finish := q.io.deq.valid +} + +object TileLinkHeaderOverwriter { + def apply[T <: ClientToManagerChannel]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { + val out = in.clone.asDirectionless + out.bits.payload := in.bits.payload + out.bits.header.src := UInt(clientId) + out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) + out.valid := in.valid + in.ready := out.ready + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + nBanks: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) + out.bits.header.dst := addrConvert(in.bits.payload.addr_block) + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val client = new HeaderlessTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) + val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.network.release <> rel_with_header + io.client.probe <> prb_without_header + io.client.grant <> gnt_without_header +} + +object TileLinkNetworkPort { + def apply[T <: Data]( + client: HeaderlessTileLinkIO, + clientId: Int = 0, + addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { + val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) + port.io.client <> client + port.io.network + } } class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { @@ -529,7 +747,6 @@ object TileLinkEnqueuer { } abstract trait HasArbiterTypes { - val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId type ClientSourcedWithIdAndData = ClientToManagerChannel with @@ -538,20 +755,19 @@ abstract trait HasArbiterTypes { } // Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule - with HasArbiterTypes { - +trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ + val arbN: Int // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -561,11 +777,11 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -575,17 +791,23 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } - arb.io.out <> mngr + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } } - def hookupManagerSourceWithId[M <: ManagerSourcedWithId] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { mngr.ready := Bool(false) for (i <- 0 until arbN) { clts(i).valid := Bool(false) @@ -594,24 +816,45 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr.ready := clts(i).ready } clts(i).bits := mngr.bits - clts(i).bits.payload.client_xact_id := - managerSourcedClientXactId(mngr.bits.payload) + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) } } - def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } } -abstract class UncachedTileLinkIOArbiter(n: Int) - extends TileLinkArbiterLike(n) { +abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -619,9 +862,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { +abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new TileLinkIO}.flip + val in = Vec.fill(arbN){new TileLinkIO}.flip val out = new TileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -631,8 +874,32 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip + val out = new HeaderlessUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip + val out = new HeaderlessTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId extends HasArbiterTypes { +trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) def managerSourcedClientXactId(in: ManagerSourcedWithId) = @@ -641,14 +908,14 @@ abstract trait AppendsArbiterId extends HasArbiterTypes { } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId extends HasArbiterTypes { +trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId extends HasArbiterTypes { +trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index de55c0e2..44981459 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,7 +2,8 @@ package uncore import Chisel._ -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] @@ -82,26 +83,24 @@ trait HasInnerTLIO extends CoherenceAgentBundle { } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new TileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def oprb(dummy: Int = 0) = outer.probe.bits.payload - def orel(dummy: Int = 0) = outer.release.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def oprb(dummy: Int = 0) = outer.probe.bits + def orel(dummy: Int = 0) = outer.release.bits + def ognt(dummy: Int = 0) = outer.grant.bits } class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { def innerTL: TileLinkIO - def outerTL: TileLinkIO + def outerTL: HeaderlessTileLinkIO def incoherent: Vec[Bool] } @@ -131,39 +130,47 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule { - def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) +abstract class XactTracker extends CoherenceAgentModule + with HasDataBeatCounters { + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + + def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + addPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - beat: UInt = UInt(0)) = { - connectDataBeatCounter(in.fire(), in.bits.payload, beat) - } - - def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - } - - def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | - ~UIntToOH(in.bits.payload.addr_beat) - } - - def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { val a = in.bits.payload - Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && - (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & - UIntToOH(a.addr_beat) + val isGetOrAtomic = a.isBuiltInType() && + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + } + + def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + dropPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } + } + + def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { + ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 14575aa5..f02cf2c3 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,15 +34,15 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) val cnt = UInt(OUTPUT, log2Up(n)) val done = Bool(OUTPUT) } - val narrowWidth = io.in.bits.payload.data.getWidth / n - require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + val narrowWidth = io.in.bits.data.getWidth / n + require(io.in.bits.data.getWidth % narrowWidth == 0) if(n == 1) { io.in <> io.out @@ -51,12 +51,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } else { val cnt = Reg(init=UInt(0, width = log2Up(n))) val wrap = cnt === UInt(n-1) - val rbits = Reg(init=io.in.bits) + val rbits = Reg(io.in.bits.clone) val active = Reg(init=Bool(false)) val shifter = Vec.fill(n){Bits(width = narrowWidth)} (0 until n).foreach { - i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } io.done := Bool(false) @@ -65,16 +65,16 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(io.in.bits.payload.hasData()) { + when(io.in.bits.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !io.in.bits.payload.hasData() + io.done := !io.in.bits.hasData() } when(active) { io.out.bits := rbits - io.out.bits.payload.data := shifter(cnt) + io.out.bits.data := shifter(cnt) when(io.out.ready) { cnt := cnt + UInt(1) when(wrap) { @@ -86,3 +86,13 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } } } + +object FlowThroughSerializer { + def apply[T <: HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { + val fs = Module(new FlowThroughSerializer(in.bits, n)) + fs.io.in.valid := in.valid + fs.io.in.bits := in.bits + in.ready := fs.io.in.ready + fs.io.out + } +} From 3cf1778c928a717e96d863330762dca14e9161a0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Apr 2015 17:24:44 -0700 Subject: [PATCH 364/374] moved ecc lib to uncore, l2 checks for partial write masks when ecc is enabled --- uncore/src/main/scala/cache.scala | 15 ++- uncore/src/main/scala/ecc.scala | 146 +++++++++++++++++++++++++++++ uncore/src/main/scala/uncore.scala | 6 +- uncore/src/main/scala/util.scala | 8 ++ 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 uncore/src/main/scala/ecc.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 05588203..492ecc0e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -15,6 +15,7 @@ case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] +case object ECCCode extends Field[Option[Code]] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -28,6 +29,7 @@ abstract trait CacheParameters extends UsesParameters { val rowBits = params(RowBits) val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) + val code = params(ECCCode).getOrElse(new IdentityCode) } abstract class CacheBundle extends Bundle with CacheParameters @@ -176,6 +178,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) val isLastLevelCache = true + val ignoresWriteMask = !params(ECCCode).isEmpty } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -462,6 +465,12 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) + + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { + val a = in.bits.payload + val isPartial = a.wmask() != Acquire.fullWriteMask + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -805,7 +814,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & dropPendingBitWhenBeatHasData(io.outer.grant)) | - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) io.data.read.valid := state === s_busy && pending_reads.orR && @@ -880,7 +890,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux( io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala new file mode 100644 index 00000000..b5864b2c --- /dev/null +++ b/uncore/src/main/scala/ecc.scala @@ -0,0 +1,146 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ + +abstract class Decoding +{ + def uncorrected: Bits + def corrected: Bits + def correctable: Bool + def uncorrectable: Bool + def error = correctable || uncorrectable +} + +abstract class Code +{ + def width(w0: Int): Int + def encode(x: Bits): Bits + def decode(x: Bits): Decoding +} + +class IdentityCode extends Code +{ + def width(w0: Int) = w0 + def encode(x: Bits) = x + def decode(y: Bits) = new Decoding { + def uncorrected = y + def corrected = y + def correctable = Bool(false) + def uncorrectable = Bool(false) + } +} + +class ParityCode extends Code +{ + def width(w0: Int) = w0+1 + def encode(x: Bits) = Cat(x.xorR, x) + def decode(y: Bits) = new Decoding { + def uncorrected = y(y.getWidth-2,0) + def corrected = uncorrected + def correctable = Bool(false) + def uncorrectable = y.xorR + } +} + +class SECCode extends Code +{ + def width(k: Int) = { + val m = new Unsigned(k).log2 + 1 + k + m + (if((1 << m) < m+k+1) 1 else 0) + } + def encode(x: Bits) = { + val k = x.getWidth + require(k > 0) + val n = width(k) + + val y = for (i <- 1 to n) yield { + if (isPow2(i)) { + val r = for (j <- 1 to n; if j != i && (j & i) != 0) + yield x(mapping(j)) + r reduce (_^_) + } else + x(mapping(i)) + } + Vec(y).toBits + } + def decode(y: Bits) = new Decoding { + val n = y.getWidth + require(n > 0 && !isPow2(n)) + + val p2 = for (i <- 0 until log2Up(n)) yield 1 << i + val syndrome = p2 map { i => + val r = for (j <- 1 to n; if (j & i) != 0) + yield y(j-1) + r reduce (_^_) + } + val s = Vec(syndrome).toBits + + private def swizzle(z: Bits) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits + def uncorrected = swizzle(y) + def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) + def correctable = s.orR + def uncorrectable = Bool(false) + } + private def mapping(i: Int) = i-1-log2Up(i) +} + +class SECDEDCode extends Code +{ + private val sec = new SECCode + private val par = new ParityCode + + def width(k: Int) = sec.width(k)+1 + def encode(x: Bits) = par.encode(sec.encode(x)) + def decode(x: Bits) = new Decoding { + val secdec = sec.decode(x(x.getWidth-2,0)) + val pardec = par.decode(x) + + def uncorrected = secdec.uncorrected + def corrected = secdec.corrected + def correctable = pardec.uncorrectable + def uncorrectable = !pardec.uncorrectable && secdec.correctable + } +} + +object ErrGen +{ + // generate a 1-bit error with approximate probability 2^-f + def apply(width: Int, f: Int): Bits = { + require(width > 0 && f >= 0 && log2Up(width) + f <= 16) + UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) + } + def apply(x: Bits, f: Int): Bits = x ^ apply(x.getWidth, f) +} + +class SECDEDTest extends Module +{ + val code = new SECDEDCode + val k = 4 + val n = code.width(k) + + val io = new Bundle { + val original = Bits(OUTPUT, k) + val encoded = Bits(OUTPUT, n) + val injected = Bits(OUTPUT, n) + val uncorrected = Bits(OUTPUT, k) + val corrected = Bits(OUTPUT, k) + val correctable = Bool(OUTPUT) + val uncorrectable = Bool(OUTPUT) + } + + val c = Counter(Bool(true), 1 << k) + val numErrors = Counter(c._2, 3)._1 + val e = code.encode(c._1) + val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1)) + val d = code.decode(i) + + io.original := c._1 + io.encoded := e + io.injected := i + io.uncorrected := d.uncorrected + io.corrected := d.corrected + io.correctable := d.correctable + io.uncorrectable := d.uncorrectable +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 44981459..62a53129 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -132,8 +132,10 @@ class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflict abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { - def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) - def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = + Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = + ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { in.bits match { diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index f02cf2c3..65c5d6cd 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -5,6 +5,14 @@ package uncore import Chisel._ import scala.math._ +class Unsigned(x: Int) { + require(x >= 0) + def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } + def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } + def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 + def nextPow2: Int = if (x == 0) 1 else 1 << clog2 +} + object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) From 90f800d87dd1d05b3837f710d9dd5d131747149e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 15:57:06 -0700 Subject: [PATCH 365/374] Grant bugfixes and more comments --- uncore/src/main/scala/cache.scala | 25 +++++++++++++++++-------- uncore/src/main/scala/tilelink.scala | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 492ecc0e..e399eb98 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) + io.inner.grant.valid := !collect_irel_data when(io.inner.grant.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -587,6 +587,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack when(io.inner.finish.valid) { state := s_idle } } } + + // Checks for illegal behavior + assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), + "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") } @@ -748,8 +752,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) - // The following subtrascation handles misses or coherence permission - // upgrades by initiating a new transaction in the outer memory: + // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -804,12 +807,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh.inner.onGrant( outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), - pending_coh.outer) + Mux(ognt_data_done, + pending_coh_on_ognt.outer, + pending_coh.outer)) updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) // We must wait for as many Finishes as we sent Grants io.inner.finish.ready := state === s_busy + // We read from the the cache at this level if data wasn't written back or refilled. + // We may merge Gets, requiring further beats to be read. + // If ECC requires a full writemask, we'll read out data on partial writes as well. pending_reads := (pending_reads & dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & @@ -829,6 +837,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { addPendingBitInternal(io.data.read) mergeDataInternal(io.data.resp) + // We write data to the cache at this level if it was Put here with allocate flag, + // written back dirty, or refilled from outer memory. pending_writes := (pending_writes & dropPendingBit(io.data.write)) | addPendingBitWhenBeatHasData(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | @@ -883,11 +893,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - pending_puts := Mux( + pending_puts := Mux( // Make sure to collect all data from a PutBlock io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenBeatHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux( + pending_reads := Mux( // GetBlocks and custom types read all beats io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | @@ -932,8 +942,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(state === s_wb_resp && io.wb.resp.valid) { // If we're overwriting the whole block in a last level cache we can // just do it without fetching any data from memory - val skip_outer_acquire = Bool(isLastLevelCache) && - xact.isBuiltInType(Acquire.putBlockType) + val skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c2bc2d30..d639f791 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -583,6 +583,7 @@ trait HasDataBeatCounters { val cnt = Reg(init = UInt(0, width = log2Up(max+1))) val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") cnt := Mux(do_dec, Mux(do_inc, cnt, cnt - UInt(1)), Mux(do_inc, cnt + UInt(1), cnt)) @@ -632,7 +633,7 @@ class FinishUnit(srcId: Int = 0) extends TLModule val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) val q = Module(new FinishQueue(entries)) - q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) q.io.enq.bits.fin := g.makeFinish() q.io.enq.bits.dst := io.grant.bits.header.src From ce3271aef24329a2bde0e5fe8f5b60e73adac564 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 19:00:40 -0700 Subject: [PATCH 366/374] refactor LNClients and LNManagers --- uncore/src/main/scala/broadcast.scala | 10 ++-- uncore/src/main/scala/cache.scala | 52 ++++++++++------- uncore/src/main/scala/memserdes.scala | 6 +- uncore/src/main/scala/network.scala | 2 - uncore/src/main/scala/tilelink.scala | 82 ++++++++++++++++----------- uncore/src/main/scala/uncore.scala | 33 ++--------- 6 files changed, 97 insertions(+), 88 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 98c8f0af..340db34d 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -187,7 +187,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_outer) { io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) when(oacq_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) + state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? } } is(s_grant) { // Forward the Grant.voluntaryAck @@ -221,13 +221,13 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e399eb98..b193c5a7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -182,7 +182,21 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { + def doInternalOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -361,8 +375,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) - doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) - doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) @@ -409,13 +423,13 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out - // Wire local memories - doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) - doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) - doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) + // Wire local memory arrays + doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doInternalOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doInternalOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doInternalOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInternalInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } @@ -620,19 +634,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = - connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) val pending_ifins = connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_meta_write = Reg{ Bool() } val all_pending_done = @@ -1006,8 +1020,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 878ce19a..942874cf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -281,7 +281,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() tl_done_out := tl_wrap_out tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -352,7 +352,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() io.mem.req_data.bits.data := io.tl.release.bits.payload.data val tag = Cat(io.tl.release.bits.header.src, io.tl.release.bits.payload.client_xact_id, @@ -366,7 +366,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) - make_grant_ack := acq_has_data + make_grant_ack := acq_has_data // i.e. is it a Put io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data val tag = Cat(io.tl.acquire.bits.header.src, diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 7fdc8e09..f61cec6c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,8 +3,6 @@ package uncore import Chisel._ -case object LNManagers extends Field[Int] -case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] case object LNHeaderBits extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d639f791..1ed95df6 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -11,25 +11,33 @@ import scala.reflect.runtime.universe._ // case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLBlockAddrBits extends Field[Int] +case object TLNManagers extends Field[Int] +case object TLNClients extends Field[Int] +case object TLNCoherentClients extends Field[Int] +case object TLNIncoherentClients extends Field[Int] case object TLMaxClientXacts extends Field[Int] case object TLMaxClientPorts extends Field[Int] case object TLMaxManagerXacts extends Field[Int] +case object TLBlockAddrBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLBlockAddrBits) + val tlCoh = params(TLCoherencePolicy) + val tlNManagers = params(TLNManagers) + val tlNClients = params(TLNClients) + val tlNCoherentClients = params(TLNCoherentClients) + val tlNIncoherentClients = params(TLNIncoherentClients) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) - val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) @@ -44,11 +52,11 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val tlNetworkDoesNotInterleaveBeats = true val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters { -} +abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel @@ -62,7 +70,7 @@ trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. -// These traits are useful for type parameterization. +// These traits are useful for type parameterizing bundle wiring functions. // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -554,10 +562,10 @@ trait HasDataBeatCounters { def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int): Vec[Bool] = { - val id = in.bits.payload.client_xact_id + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 }) } @@ -615,41 +623,51 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0) extends TLModule +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) val finish = Decoupled(new LogicalNetworkIO(new Finish)) val ready = Bool(OUTPUT) - val grant_done = Bool(OUTPUT) - val pending_finish = Bool(OUTPUT) } - val entries = 1 << tlClientXactIdBits val g = io.grant.bits.payload - assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) - val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) - val q = Module(new FinishQueue(entries)) + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounter(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits.fin := g.makeFinish() - q.io.enq.bits.dst := io.grant.bits.header.src + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.dst - io.finish.bits.payload := q.io.deq.bits.fin - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := io.grant.valid - io.refill.bits := io.grant.bits.payload - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) - - io.ready := q.io.enq.ready - io.grant_done := done - io.pending_finish := q.io.deq.valid + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } } object TileLinkHeaderOverwriter { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 62a53129..9d1c93b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -16,9 +16,6 @@ trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nCoherentClients = params(NCoherentClients) - val nIncoherentClients = params(NIncoherentClients) - val nClients = nCoherentClients + nIncoherentClients def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) @@ -35,33 +32,15 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: Data : ClassTag]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( - out: DecoupledIO[S], - ins: Seq[DecoupledIO[S]]) { + def doOutputArbitration[T <: TileLinkChannel : ClassTag]( + out: DecoupledIO[LogicalNetworkIO[T]], + ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( - out.bits.clone, - ins.size, - out.bits.payload.tlDataBeats, - lock _)) + val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } - } - def doInputRouting[T <: HasManagerTransactionId]( in: DecoupledIO[LogicalNetworkIO[T]], outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { @@ -74,7 +53,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload @@ -173,6 +152,6 @@ abstract class XactTracker extends CoherenceAgentModule } def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) + ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } From ba7a8b175207e6712efddd7c4683e052cb8691c3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 17 Apr 2015 16:55:20 -0700 Subject: [PATCH 367/374] TileLink refactor; TileLinkPorts now available. L2Banks no longer have unique ids (suitable for hierarhical P&R). --- uncore/src/main/scala/broadcast.scala | 54 ++-- uncore/src/main/scala/cache.scala | 132 ++++---- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/memserdes.scala | 185 ++++++----- uncore/src/main/scala/metadata.scala | 41 ++- uncore/src/main/scala/tilelink.scala | 427 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 84 ++--- 7 files changed, 460 insertions(+), 465 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 340db34d..339a7540 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -25,16 +25,16 @@ object DataQueueLocation { } } -class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent +class L2BroadcastHub extends ManagerCoherenceAgent with BroadcastHubParameters { val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.incoherent := io.incoherent.toBits) @@ -61,7 +61,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.bits.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } @@ -82,7 +82,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent case(tracker, i) => tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits - tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + tracker.bits.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits } @@ -91,17 +91,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat + io.inner.grant.bits.data := io.outer.grant.bits.data + io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue @@ -124,12 +124,11 @@ class BroadcastXactTracker extends XactTracker { val io = new ManagerXactTrackerIO } -class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val coh = ManagerMetadata.onReset @@ -150,9 +149,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? io.outer.acquire.bits := Bundle( @@ -175,7 +172,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(UInt(0)) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -207,12 +203,11 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa } } -class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } val coh = ManagerMetadata.onReset @@ -225,9 +220,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val probe_self = io.inner.acquire.bits.requiresSelfProbe() + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits @@ -272,21 +267,17 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := coh.makeProbe(xact) + io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.iacq().client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && @@ -316,7 +307,6 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact is(s_idle) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193c5a7..8ae4d00b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -188,7 +188,7 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { @@ -342,14 +342,14 @@ class L2SecondaryMissInfo extends TLBundle with HasTileLinkBeatId with HasClientTransactionId -class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) val meta = Module(new L2MetadataArray) // TODO: add delay knob val data = Module(new L2DataArray(1)) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile) tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -362,24 +362,21 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile(bankId: Int) extends L2HellaCacheModule +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) - } + val trackerList = (0 until nReleaseTransactors).map(id => Module(new L2VoluntaryReleaseTracker(id))) ++ + (nReleaseTransactors until nTransactors).map(id => Module(new L2AcquireTracker(id))) // WritebackUnit evicts data from L2, including invalidating L1s - val wb = Module(new L2WritebackUnit(nTransactors, bankId)) + val wb = Module(new L2WritebackUnit(nTransactors)) doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent.toBits } // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) @@ -419,8 +416,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) - outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size))(outerTLParams) + outer_arb.io.in <> outerList io.outer <> outer_arb.io.out // Wire local memory arrays @@ -480,21 +477,20 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) - def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isPartial = a.wmask() != Acquire.fullWriteMask - addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -520,9 +516,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) @@ -541,7 +535,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) @@ -558,7 +552,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(io.irel().addr_beat) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -608,15 +601,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) // State holding transaction metadata - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } @@ -631,16 +623,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // TODO add ignt.dst <- iacq.src // State holding progress made on processing this transaction - val iacq_data_done = - connectIncomingDataBeatCounter(io.inner.acquire) - val pending_irels = - connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = connectTwoWayBeatCounter( + max = io.inner.tlNCoherentClients, + up = io.inner.probe, + down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = - connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) - val (ignt_data_idx, ignt_data_done) = - connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - val pending_ifins = - connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + connectTwoWayBeatCounter( + max = 1, + up = io.outer.acquire, + down = io.outer.grant, + beat = xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = connectTwoWayBeatCounter( + max = nSecondaryMisses, + up = io.inner.grant, + down = io.inner.finish, + track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -695,9 +694,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } - def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { - when(in.fire() && in.bits.payload.hasData()) { - mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) } } def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { @@ -713,7 +712,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -728,7 +727,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -749,17 +748,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data io.inner.release.ready := state === s_inner_probe val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( // Drop sharer - incoming = io.irel(), - src = io.inner.release.bits.header.src), + pending_coh.inner.onRelease(io.irel()), // Drop sharer Mux(io.irel().hasData(), // Dirty writeback pending_coh.outer.onHit(M_XWR), pending_coh.outer)) @@ -804,23 +799,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of - // multiple client transaction ids from merged secondary misses - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + io.inner.grant.bits := pending_coh.inner.makeGrant( + dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src + acq = xact, + client_xact_id = ignt_q.io.deq.bits.client_xact_id, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), + pending_coh.inner.onGrant(io.ignt()), Mux(ognt_data_done, pending_coh_on_ognt.outer, pending_coh.outer)) @@ -904,7 +894,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) pending_puts := Mux( // Make sure to collect all data from a PutBlock @@ -943,8 +932,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val full_sharers = coh.inner.full() val mask_self = Mux( xact.requiresSelfProbe(), - coh.inner.full() | UIntToOH(xact_src), - coh.inner.full() & ~UIntToOH(xact_src)) + coh.inner.full() | UIntToOH(xact.client_id), + coh.inner.full() & ~UIntToOH(xact.client_id)) val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } @@ -984,7 +973,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.inner.acquire.bits.client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") } @@ -1007,7 +996,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { val data = new L2DataRWIO } -class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) @@ -1031,9 +1020,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) + val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write val pending_ofin_on_ognt = io.ognt().makeFinish() @@ -1054,10 +1041,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := - xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) + io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 787847b9..8a79cda1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new HeaderlessUncachedTileLinkIO + val mem = new ClientUncachedTileLinkIO val scr = new SCRIO } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 942874cf..7497e2e4 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -18,16 +18,26 @@ case object MIFDataBits extends Field[Int] case object MIFTagBits extends Field[Int] case object MIFDataBeats extends Field[Int] -trait HasMemData extends Bundle { - val data = Bits(width = params(MIFDataBits)) +trait MIFParameters extends UsesParameters { + val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) +} + +abstract class MIFBundle extends Bundle with MIFParameters +abstract class MIFModule extends Module with MIFParameters + +trait HasMemData extends MIFBundle { + val data = Bits(width = mifDataBits) } -trait HasMemAddr extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) +trait HasMemAddr extends MIFBundle { + val addr = UInt(width = mifAddrBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = params(MIFTagBits)) +trait HasMemTag extends MIFBundle { + val tag = UInt(width = mifTagBits) } class MemReqCmd extends HasMemAddr with HasMemTag { @@ -56,7 +66,7 @@ class MemSerializedIO(w: Int) extends Bundle val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int) extends MIFModule { val io = new Bundle { val wide = new MemIO().flip @@ -65,7 +75,6 @@ class MemSerdes(w: Int) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -73,7 +82,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -109,12 +118,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(mifDataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -143,11 +152,11 @@ class MemDesser(w: Int) extends Module // test rig side val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) + val mifDataBeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -175,13 +184,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -195,7 +204,7 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, dbeats)) + val dataq = Module(new Queue(new MemResp, mifDataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -203,19 +212,12 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } -//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary - - //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemIO } - val mifTagBits = params(MIFTagBits) - val mifAddrBits = params(MIFAddrBits) - val mifDataBits = params(MIFDataBits) - val mifDataBeats = params(MIFDataBeats) val dataBits = tlDataBits*tlDataBeats val dstIdBits = params(LNHeaderBits) require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") @@ -227,12 +229,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.finish.ready := Bool(true) io.mem.resp.ready := Bool(false) - val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.payload.hasData() - val rel_has_data = io.tl.release.bits.payload.hasData() + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) @@ -248,13 +250,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val make_grant_ack = Reg(init=Bool(false)) gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Grant( - is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> UInt(1), - manager_xact_id = UInt(0)) - gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(1).bits.header.src := UInt(0) + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -263,7 +264,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { mem_data_q.io.enq.valid := Bool(false) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.data.clone) } val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) @@ -275,27 +276,27 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - addr_out := io.tl.release.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + addr_out := io.tl.release.bits.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() + make_grant_ack := io.tl.release.bits.requiresAck() tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.release.bits.data } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - addr_out := io.tl.acquire.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + addr_out := io.tl.acquire.bits.addr_block has_data := acq_has_data data_from_rel := Bool(false) make_grant_ack := acq_has_data tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data } } when(active_out) { @@ -306,8 +307,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.acquire.ready := Bool(true) when(io.tl.acquire.valid) { tl_buf_out(tl_cnt_out) := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) } } when(!mif_done_out) { @@ -338,8 +339,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.mem.req_cmd.bits.tag := tag_out io.mem.req_cmd.bits.addr := addr_out io.mem.req_data.bits.data := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) when(!active_out){ io.tl.release.ready := io.mem.req_data.ready io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid @@ -352,12 +353,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() - io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - val addr = io.tl.release.bits.payload.addr_block + make_grant_ack := io.tl.release.bits.requiresAck() + io.mem.req_data.bits.data := io.tl.release.bits.data + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := rel_has_data @@ -367,12 +368,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data // i.e. is it a Put - io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_data.bits.data := io.tl.acquire.bits.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - val addr = io.tl.acquire.bits.payload.addr_block + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := acq_has_data @@ -407,28 +408,25 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { // Aggregate incoming MemIO responses into TL Grants val active_in = Reg(init=Bool(false)) - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = tag_in(0), - g_type = Mux(tag_in(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -453,17 +451,14 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = io.mem.resp.bits.tag(0), - g_type = Mux(io.mem.resp.bits.tag(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } @@ -522,28 +517,28 @@ object HellaQueue } } -class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { +class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } - val numEntries = numRequests * refillCycles + val numEntries = numRequests * mifDataBeats val size = log2Down(numEntries) + 1 val inc = Bool() val dec = Bool() val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) + val watermark = count >= UInt(mifDataBeats) when (inc && !dec) { count := count + UInt(1) } when (!inc && dec) { - count := count - UInt(refillCycles) + count := count - UInt(mifDataBeats) } when (inc && dec) { - count := count - UInt(refillCycles-1) + count := count - UInt(mifDataBeats-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -573,17 +568,17 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemPipeIO } val a = Module(new MemIOTileLinkIOConverter(1)) - val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + val b = Module(new MemPipeIOMemIOConverter(outstanding)) a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index af5504c8..4e8082ee 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -108,17 +108,18 @@ class ManagerMetadata extends CoherenceMetadata { def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(cmd: UInt, addr_block: UInt): Probe = - Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - def makeProbe(acq: Acquire): Probe = - Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) - def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = - makeProbe(M_FLUSH, addr_block) + def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = + makeProbe(dst, M_FLUSH, addr_block) - def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( + dst = rel.client_id, is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, @@ -126,11 +127,12 @@ class ManagerMetadata extends CoherenceMetadata { } def makeGrant( - acq: Acquire, + acq: AcquireFromSrc, manager_xact_id: UInt, addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + data: UInt = UInt(0)): GrantToDst = { Bundle(Grant( + dst = acq.client_id, is_builtin_type = acq.isBuiltInType(), g_type = Mux(acq.isBuiltInType(), acq.getBuiltInGrantType(), @@ -141,11 +143,24 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } - def onRelease(incoming: Release, src: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + def makeGrant( + dst: UInt, + acq: AcquireFromSrc, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt, + data: UInt): GrantToDst = { + val g = makeGrant(acq, manager_xact_id, addr_beat, data) + g.client_id := dst + g.client_xact_id := client_xact_id + g + } + + def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) - def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) + def onGrant(outgoing: GrantToDst): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } object ManagerMetadata { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1ed95df6..45cdb454 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -23,12 +23,14 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] -abstract trait TileLinkParameters extends UsesParameters { +trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) val tlNCoherentClients = params(TLNCoherentClients) val tlNIncoherentClients = params(TLNIncoherentClients) + val tlClientIdBits = log2Up(tlNClients) + val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) @@ -98,6 +100,10 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +trait HasClientId extends TLBundle { + val client_id = UInt(width = tlClientIdBits) +} + // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel @@ -350,12 +356,19 @@ class Probe extends ManagerToClientChannel } object Probe { - def apply(p_type: UInt, addr_block: UInt) = { + def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } + def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { + val prb = new ProbeToDst + prb.client_id := dst + prb.p_type := p_type + prb.addr_block := addr_block + prb + } } class Release extends ClientToManagerChannel @@ -435,8 +448,8 @@ object Grant { g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + addr_beat: UInt, + data: UInt): Grant = { val gnt = new Grant gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type @@ -446,6 +459,25 @@ object Grant { gnt.data := data gnt } + + def apply( + dst: UInt, + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = { + val gnt = new GrantToDst + gnt.client_id := dst + gnt.is_builtin_type := is_builtin_type + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat + gnt.data := data + gnt + } } class Finish extends ClientToManagerChannel with HasManagerTransactionId { @@ -453,7 +485,14 @@ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// Complete IO definitions for two types of TileLink clients +// These subtypes include a field for the source or destination ClientId +class AcquireFromSrc extends Acquire with HasClientId +class ProbeToDst extends Probe with HasClientId +class ReleaseFromSrc extends Release with HasClientId +class GrantToDst extends Grant with HasClientId + +// Complete IO definitions for two types of TileLink clients, including +// networking headers class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -479,28 +518,36 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } -// This version of TileLinkIO does not contain network headers for packets -// that originate in the Clients (i.e. Acquire and Release). These headers -// are provided in the top-level that instantiates the clients and network. +// This version of TileLinkIO does not contain network headers. The headers +// are provided in the top-level that instantiates the clients and network, +// probably using a TileLinkClientPort module. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. // Secondly, this version of the interface elides Finish messages, with the // assumption that a FinishUnit has been coupled to the TileLinkIO port // to deal with acking received Grants. -class HeaderlessUncachedTileLinkIO extends TLBundle { +class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } -class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { +class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } -class HeaderlessTileLinkIOWrapper extends TLModule { +class ManagerTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new AcquireFromSrc).flip + val grant = new DecoupledIO(new GrantToDst) + val finish = new DecoupledIO(new Finish).flip + val probe = new DecoupledIO(new ProbeToDst) + val release = new DecoupledIO(new ReleaseFromSrc).flip +} + +class ClientTileLinkIOWrapper extends TLModule { val io = new Bundle { - val in = new HeaderlessUncachedTileLinkIO().flip - val out = new HeaderlessTileLinkIO + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant @@ -509,17 +556,17 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } object TileLinkIOWrapper { - def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper)(p) conv.io.in <> utl conv.io.out } - def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper) + def apply(utl: ClientUncachedTileLinkIO): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) conv.io.in <> utl conv.io.out } - def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { val conv = Module(new TileLinkIOWrapper)(p) conv.io.in <> utl @@ -533,89 +580,6 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - type HasClientId = TileLinkChannel with HasClientTransactionId - type LNAcquire = LogicalNetworkIO[Acquire] - type LNRelease = LogicalNetworkIO[Release] - type LNGrant = LogicalNetworkIO[Grant] - - def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( - in: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, beat) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, beat) - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} - } - } - - def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, UInt(0))._2 - case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} - } - } - - def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[T], - down: DecoupledIO[S], - beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) - val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) - //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) - } - - def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[LogicalNetworkIO[T]], - down: DecoupledIO[LogicalNetworkIO[S]], - inc: T => Bool = (t: T) => Bool(true), - dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) - val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) - val do_inc = up_done && inc(up.bits.payload) - val do_dec = down_done && dec(down.bits.payload) - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, up_done, down_idx, down_done) - } -} - class FinishQueueEntry extends TLBundle { val fin = new Finish val dst = UInt(width = log2Up(params(LNEndpoints))) @@ -623,8 +587,7 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule - with HasDataBeatCounters { +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) @@ -645,12 +608,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule // a multibeat Grant. But Grants from multiple managers or transactions may // get interleaved, so we could need a counter for each. val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounter(io.grant) + connectIncomingDataBeatCounterWithHeader(io.grant) } else { val entries = 1 << tlClientXactIdBits def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) } val q = Module(new FinishQueue(outstanding)) q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) @@ -670,30 +633,7 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule } } -object TileLinkHeaderOverwriter { - def apply[T <: ClientToManagerChannel]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { - val out = in.clone.asDirectionless - out.bits.payload := in.bits.payload - out.bits.header.src := UInt(clientId) - out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) - out.valid := in.valid - in.ready := out.ready - out - } - - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - nBanks: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) - out.bits.header.dst := addrConvert(in.bits.payload.addr_block) - out - } - +object ClientTileLinkHeaderCreator { def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( in: DecoupledIO[T], clientId: Int, @@ -708,9 +648,24 @@ object TileLinkHeaderOverwriter { } } -class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { - val client = new HeaderlessTileLinkIO().flip + val client = new ClientTileLinkIO().flip val network = new TileLinkIO } @@ -718,8 +673,8 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo finisher.io.grant <> io.network.grant io.network.finish <> finisher.io.finish - val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) - val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) val gnt_without_header = finisher.io.refill @@ -731,58 +686,63 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo io.client.grant <> gnt_without_header } -object TileLinkNetworkPort { - def apply[T <: Data]( - client: HeaderlessTileLinkIO, - clientId: Int = 0, - addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { - val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) - port.io.client <> client - port.io.network +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } -class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip val manager = new TileLinkIO } - io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) - io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) - io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) - io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) - io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) } object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + def apply(in: TileLinkIO, depths: TileLinkDepths)(p: Parameters): TileLinkIO = { val t = Module(new TileLinkEnqueuer(depths))(p) t.io.client <> in t.io.manager } def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { - apply(in, (depth, depth, depth, depth, depth))(p) + apply(in, TileLinkDepths(depth, depth, depth, depth, depth))(p) } } -abstract trait HasArbiterTypes { +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends TileLinkParameters { + // Some shorthand type variables type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId - type ClientSourcedWithIdAndData = ClientToManagerChannel with - HasClientTransactionId with - HasTileLinkData -} + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData -// Utility functions for constructing TileLinkIO arbiters -trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ - val arbN: Int - // These are filled in depending on whether the arbiter mucks with the - // client ids and then needs to revert them on the way back + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() @@ -796,7 +756,7 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() @@ -854,23 +814,20 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ } } - def hookupManagerSourceBroadcast[M <: Data]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.in <> clts arb.io.out <> mngr } } +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip @@ -881,6 +838,7 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with Tile hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new TileLinkIO}.flip @@ -893,31 +851,7 @@ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbi hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip - val out = new HeaderlessUncachedTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip - val out = new HeaderlessTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -// Appends the port index of the arbiter to the client_xact_id +/** Appends the port index of the arbiter to the client_xact_id */ trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) @@ -926,24 +860,125 @@ trait AppendsArbiterId extends TileLinkArbiterLike { def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } -// Uses the client_xact_id as is (assumes it has been set to port index) +/** Uses the client_xact_id as is (assumes it has been set to port index) */ trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Overwrites some default client_xact_id with the port idx +/** Overwrites some default client_xact_id with the port idx */ trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Mix-in id generation traits to make concrete arbiter classes +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientUncachedTileLinkIO}.flip + val out = new ClientUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientTileLinkIO}.flip + val out = new ClientTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular TileLinkChannel or pair of channels. + * + * Won't count message types that don't have data. + * Used in XactTrackers and FinishUnit. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing DecoupledIOs */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(in.fire(), in.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param max max number of outstanding ups with no down + * @param up outgoing channel + * @param down incoming channel + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt = UInt(0), + track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) + val do_inc = up_done && track(up.bits) + val do_dec = down_done + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 9d1c93b7..8dda15b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -32,19 +32,19 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: TileLinkChannel : ClassTag]( - out: DecoupledIO[LogicalNetworkIO[T]], - ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) + def doOutputArbitration[T <: TileLinkChannel]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + def lock(o: T) = o.hasMultibeatData() + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, out.bits.tlDataBeats, lock _)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInputRouting[T <: HasManagerTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - val idx = in.bits.payload.manager_xact_id + in: DecoupledIO[T], + outs: Seq[DecoupledIO[T]]) { + val idx = in.bits.manager_xact_id outs.map(_.bits := in.bits) outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } in.ready := Vec(outs.map(_.ready)).read(idx) @@ -52,23 +52,23 @@ trait HasCoherenceAgentWiringHelpers { } trait HasInnerTLIO extends CoherenceAgentBundle { - val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput - def iacq(dummy: Int = 0) = inner.acquire.bits.payload - def iprb(dummy: Int = 0) = inner.probe.bits.payload - def irel(dummy: Int = 0) = inner.release.bits.payload - def ignt(dummy: Int = 0) = inner.grant.bits.payload - def ifin(dummy: Int = 0) = inner.finish.bits.payload + def iacq(dummy: Int = 0) = inner.acquire.bits + def iprb(dummy: Int = 0) = inner.probe.bits + def irel(dummy: Int = 0) = inner.release.bits + def ignt(dummy: Int = 0) = inner.grant.bits + def ifin(dummy: Int = 0) = inner.finish.bits } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits def orel(dummy: Int = 0) = outer.release.bits @@ -78,8 +78,8 @@ trait HasCachedOuterTLIO extends CoherenceAgentBundle { class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { - def innerTL: TileLinkIO - def outerTL: HeaderlessTileLinkIO + def innerTL: ManagerTileLinkIO + def outerTL: ClientTileLinkIO def incoherent: Vec[Bool] } @@ -109,49 +109,25 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule - with HasDataBeatCounters { +abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) - def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - addPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isGetOrAtomic = a.isBuiltInType() && - (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) - addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) } - def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - dropPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) - } + def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = + ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From 6d40a610602e3250cd247cacb5d84722914f01d5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 19 Apr 2015 22:06:44 -0700 Subject: [PATCH 368/374] TileLink scala doc and parameter renaming --- uncore/src/main/scala/broadcast.scala | 8 +- uncore/src/main/scala/cache.scala | 8 +- uncore/src/main/scala/metadata.scala | 10 +- uncore/src/main/scala/tilelink.scala | 500 +++++++++++++++++++------- uncore/src/main/scala/uncore.scala | 10 +- 5 files changed, 381 insertions(+), 155 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 339a7540..b554b231 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -216,13 +216,13 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8ae4d00b..75bb9e1f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -625,7 +625,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = connectTwoWayBeatCounter( - max = io.inner.tlNCoherentClients, + max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = @@ -641,7 +641,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { down = io.inner.finish, track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -1009,8 +1009,8 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 4e8082ee..cf52d286 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -9,11 +9,11 @@ abstract class CoherenceMetadata extends Bundle { val id = params(TLId) } -/* The ClientMetadata stores the client-side coherence information, - such as permissions on the data and whether the data is dirty. - Its API can be used to make TileLink messages in response to - memory operations or TileLink Probes. -*/ +/** Stores the client-side coherence information, + * such as permissions on the data and whether the data is dirty. + * Its API can be used to make TileLink messages in response to + * memory operations or [[uncore.Probe]] messages. + */ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45cdb454..b759d48b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,38 +3,50 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect._ -import scala.reflect.runtime.universe._ -// Parameters exposed to the top-level design, set based on -// external requirements or design space exploration -// -case object TLId extends Field[String] // Unique name per network +/** Parameters exposed to the top-level design, set based on + * external requirements or design space exploration + */ +/** Unique name per TileLink network*/ +case object TLId extends Field[String] +/** Coherency policy used to define custom mesage types */ case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLNManagers extends Field[Int] +/** Number of manager agents */ +case object TLNManagers extends Field[Int] +/** Number of client agents */ case object TLNClients extends Field[Int] -case object TLNCoherentClients extends Field[Int] -case object TLNIncoherentClients extends Field[Int] +/** Number of client agents that cache data and use custom [[uncore.Acquire]] types */ +case object TLNCachingClients extends Field[Int] +/** Number of client agents that do not cache data and use built-in [[uncoreAcquire]] types */ +case object TLNCachelessClients extends Field[Int] +/** Maximum number of unique outstanding transactions per client */ case object TLMaxClientXacts extends Field[Int] -case object TLMaxClientPorts extends Field[Int] +/** Maximum number of clients multiplexed onto a single port */ +case object TLMaxClientsPerPort extends Field[Int] +/** Maximum number of unique outstanding transactions per manager */ case object TLMaxManagerXacts extends Field[Int] +/** Width of cache block addresses */ case object TLBlockAddrBits extends Field[Int] +/** Width of data beats */ case object TLDataBits extends Field[Int] +/** Number of data beats per cache block */ case object TLDataBeats extends Field[Int] +/** Whether the underlying physical network preserved point-to-point ordering of messages */ case object TLNetworkIsOrderedP2P extends Field[Boolean] +/** Utility trait for building Modules and Bundles that use TileLink parameters */ trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) - val tlNCoherentClients = params(TLNCoherentClients) - val tlNIncoherentClients = params(TLNIncoherentClients) + val tlNCachingClients = params(TLNCachingClients) + val tlNCachelessClients = params(TLNCachelessClients) val tlClientIdBits = log2Up(tlNClients) val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) - val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxClientsPerPort = params(TLMaxClientsPerPort) val tlMaxManagerXacts = params(TLMaxManagerXacts) - val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) @@ -61,19 +73,23 @@ trait TileLinkParameters extends UsesParameters { abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters -// Directionality of message channel -// Used to hook up logical network ports to physical network ports +/** Base trait for all TileLink channels */ trait TileLinkChannel extends TLBundle { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToManagerChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ManagerToClientChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToClientChannel extends TileLinkChannel // Unused for now -// Common signals that are used in multiple channels. -// These traits are useful for type parameterizing bundle wiring functions. -// +/** Common signals that are used in multiple channels. + * These traits are useful for type parameterizing bundle wiring functions. + */ + +/** Address of a cache block. */ trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -81,18 +97,22 @@ trait HasCacheBlockAddress extends TLBundle { def conflicts(addr: UInt) = this.addr_block === addr } +/** Sub-block address or beat id of multi-beat data */ trait HasTileLinkBeatId extends TLBundle { val addr_beat = UInt(width = tlBeatAddrBits) } +/* Client-side transaction id. Usually Miss Status Handling Register File index */ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } +/** Manager-side transaction id. Usually Transaction Status Handling Register File index. */ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } +/** A single beat of cache block data */ trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) @@ -100,61 +120,90 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +/** The id of a client source or destination. Used in managers. */ trait HasClientId extends TLBundle { val client_id = UInt(width = tlClientIdBits) } -// Actual TileLink channel bundle definitions +/** TileLink channel bundle definitions */ +/** The Acquire channel is used to intiate coherence protocol transactions in + * order to gain access to a cache block's data with certain permissions + * enabled. Messages sent over this channel may be custom types defined by + * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types + * used for uncached data accesses. Acquires may contain data for Put or + * PutAtomic built-in types. After sending an Acquire, clients must + * wait for a manager to send them a [[uncore.Grant]] message in response. + */ class Acquire extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - // Actual bundle fields + // Actual bundle fields: val is_builtin_type = Bool() val a_type = UInt(width = tlAcquireTypeBits) val union = Bits(width = tlAcquireUnionBits) - // Utility funcs for accessing subblock union + // Utility funcs for accessing subblock union: val opCodeOff = 1 val opSizeOff = tlMemoryOpcodeBits + opCodeOff val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff + /** Hint whether to allocate the block in any interveneing caches */ def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), + /** Op code for [[uncore.PutAtomic]] operations */ + def op_code(dummy: Int = 0) = Mux( + isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) + /** Operand size for [[uncore.PutAtomic]] */ def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + /** Bit offset of [[uncore.PutAtomic]] operand */ def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) + /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) - + /** Complete physical address for block, beat or operand */ def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) - // Other helper funcs + // Other helper functions: + /** Message type equality */ def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? + /** Is this message a built-in or custom type */ def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + /** Is this message a particular built-in type */ def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t + /** Does this message refer to subblock operands using info in the Acquire.union subbundle */ def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + /** Is this message a built-in prefetch message */ def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) - // Assumes no custom types have data + /** Does this message contain data? Assumes that no custom message types have data. */ def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + /** Does this message contain multiple beats of data? Assumes that no custom message types have data. */ def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) + /** Does this message require the manager to probe the client the very client that sent it? + * Needed if multiple caches are attached to the same port. + */ def requiresSelfProbe(dummy: Int = 0) = Bool(false) + /** Mapping between each built-in Acquire type (defined in companion object) + * and a built-in Grant type. + */ def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( Acquire.getType -> Grant.getDataBeatType, @@ -166,15 +215,33 @@ class Acquire extends ClientToManagerChannel } } +/** [[uncore.Acquire]] with an extra field stating its source id */ +class AcquireFromSrc extends Acquire with HasClientId + +/** Contains definitions of the the built-in Acquire types and a factory + * for [[uncore.Acquire]] + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeAcquire]] for custom cached Acquires and + * [[uncore.Get]], [[uncore.Put]], etc. for built-in uncached Acquires. + * + * @param is_builtin_type built-in or custom type message? + * @param a_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being put outwards + * @param union additional fields used for uncached types + */ object Acquire { val nBuiltInTypes = 5 //TODO: Use Enum - def getType = UInt("b000") - def getBlockType = UInt("b001") - def putType = UInt("b010") - def putBlockType = UInt("b011") - def putAtomicType = UInt("b100") - def prefetchType = UInt("b101") + def getType = UInt("b000") // Get a single beat of data + def getBlockType = UInt("b001") // Get a whole block of data + def putType = UInt("b010") // Put a single beat of data + def putBlockType = UInt("b011") // Put a whole block of data + def putAtomicType = UInt("b100") // Perform an atomic memory op + def prefetchType = UInt("b101") // Prefetch a whole block of data def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -208,7 +275,16 @@ object Acquire { } } -// Asks for a single TileLink beat of data +/** Get a single beat of data from the outer memory hierarchy + * + * The client can hint whether he block containing this beat should be + * allocated in the intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param alloc hint whether the block should be allocated in intervening caches + */ object Get { def apply( client_xact_id: UInt, @@ -225,7 +301,15 @@ object Get { } } -// Asks for an entire cache block of data +/** Get a whole cache block of data from the outer memory hierarchy + * + * The client can hint whether the block should be allocated in the + * intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param alloc hint whether the block should be allocated in intervening caches + */ object GetBlock { def apply( client_xact_id: UInt = UInt(0), @@ -240,8 +324,12 @@ object GetBlock { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with read permissions +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with read permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ object GetPrefetch { def apply( client_xact_id: UInt, @@ -256,7 +344,16 @@ object GetPrefetch { } } -// Writes up to a single TileLink beat of data, using mask +/** Put a single beat of data into the outer memory hierarchy + * + * The block will be allocated in the next-outermost level of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + */ object Put { def apply( client_xact_id: UInt, @@ -275,7 +372,19 @@ object Put { } } -// Writes an entire cache block of data +/** Put a whole cache block of data into the outer memory hierarchy + * + * If the write mask is not full, the block will be allocated in the + * next-outermost level of the hierarchy. If the write mask is full, the + * client can hint whether the block should be allocated or not. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat of several) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + * @param alloc hint whether the block should be allocated in intervening caches + */ object PutBlock { def apply( client_xact_id: UInt, @@ -309,7 +418,36 @@ object PutBlock { } } -// Performs an atomic operation in the outer memory +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with write permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + +/** Perform an atomic memory operation in the next-outermost level of the memory hierarchy + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (within which beat) + * @param addr_byte sub-block address (which byte) + * @param atomic_opcode {swap, add, xor, and, min, max, minu, maxu} from [[uncore.MemoryOpConstants]] + * @param operand_size {byte, half, word, double} from [[uncore.MemoryOpConstants]] + * @param data source operand data + */ object PutAtomic { def apply( client_xact_id: UInt, @@ -330,22 +468,11 @@ object PutAtomic { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with write permissions -object PutPrefetch { - def apply( - client_xact_id: UInt, - addr_block: UInt): Acquire = { - Acquire( - is_builtin_type = Bool(true), - a_type = Acquire.prefetchType, - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = UInt(0), - union = Cat(M_XWR, Bool(true))) - } -} - +/** The Probe channel is used to force clients to release data or cede permissions + * on a cache block. Clients respond to Probes with [[uncore.Release]] messages. + * The available types of Probes are customized by a particular + * [[uncore.CoherencePolicy]]. + */ class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) @@ -355,6 +482,18 @@ class Probe extends ManagerToClientChannel def hasMultibeatData(dummy: Int = 0) = Bool(false) } +/** [[uncore.Probe]] with an extra field stating its destination id */ +class ProbeToDst extends Probe with HasClientId + +/** Contains factories for [[uncore.Probe]] and [[uncore.ProbeToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeProbe]] instead. + * + * @param dst id of client to which probe should be sent + * @param p_type custom probe type + * @param addr_block address of the cache block + */ object Probe { def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe @@ -371,6 +510,13 @@ object Probe { } } +/** The Release channel is used to release data or permission back to the manager + * in response to [[uncore.Probe]] messages. It can also be used to voluntarily + * write back data, for example in the event that dirty data must be evicted on + * a cache miss. The available types of Release messages are always customized by + * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be + * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grants]]. + */ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -387,6 +533,21 @@ class Release extends ClientToManagerChannel def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } +/** [[uncore.Release]] with an extra field stating its source id */ +class ReleaseFromSrc extends Release with HasClientId + +/** Contains a [[uncore.Release]] factory + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeRelease]] instead. + * + * @param voluntary is this a voluntary writeback + * @param r_type type enum defined by coherence protocol + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat beat id of the data + * @param data data being written back + */ object Release { def apply( voluntary: Bool, @@ -406,6 +567,13 @@ object Release { } } +/** The Grant channel is used to refill data or grant permissions requested of the + * manager agent via an [[uncore.Acquire]] message. It is also used to acknowledge + * the receipt of voluntary writeback from clients in the form of [[uncore.Release]] + * messages. There are built-in Grant messages used for Gets and Puts, and + * coherence policies may also define custom Grant types. Grants may contain data + * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. + */ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId @@ -433,13 +601,30 @@ class Grant extends ManagerToClientChannel } } +/** [[uncore.Grant]] with an extra field stating its destination */ +class GrantToDst extends Grant with HasClientId + +/** Contains definitions of the the built-in grant types and factories + * for [[uncore.Grant]] and [[uncore.GrantToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeGrant]] instead. + * + * @param dst id of client to which grant should be sent + * @param is_builtin_type built-in or custom type message? + * @param g_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ object Grant { val nBuiltInTypes = 5 - def voluntaryAckType = UInt("b000") - def putAckType = UInt("b001") - def prefetchAckType = UInt("b011") - def getDataBeatType = UInt("b100") - def getDataBlockType = UInt("b101") + def voluntaryAckType = UInt("b000") // For acking Releases + def prefetchAckType = UInt("b001") // For acking any kind of Prefetch + def putAckType = UInt("b011") // For acking any kind of non-prfetch Put + def getDataBeatType = UInt("b100") // Supplying a single beat of Get + def getDataBlockType = UInt("b101") // Supplying all beats of a GetBlock def typesWithData = Vec(getDataBlockType, getDataBeatType) def typesWithMultibeatData= Vec(getDataBlockType) @@ -480,62 +665,69 @@ object Grant { } } +/** The Finish channel is used to provide a global ordering of transactions + * in networks that do not guarantee point-to-point ordering of messages. + * A Finsish message is sent as acknowledgement of receipt of a [[uncore.Grant]]. + * When a Finish message is received, a manager knows it is safe to begin + * processing other transactions that touch the same cache block. + */ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasData(dummy: Int = 0) = Bool(false) def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// These subtypes include a field for the source or destination ClientId -class AcquireFromSrc extends Acquire with HasClientId -class ProbeToDst extends Probe with HasClientId -class ReleaseFromSrc extends Release with HasClientId -class GrantToDst extends Grant with HasClientId - -// Complete IO definitions for two types of TileLink clients, including -// networking headers +/** Complete IO definition for incoherent TileLink, including networking headers */ class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } +/** Complete IO definition for coherent TileLink, including networking headers */ class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -// Converts UncachedTileLinkIO to regular TileLinkIO by pinning -// probe.ready and release.valid low -class TileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -// This version of TileLinkIO does not contain network headers. The headers -// are provided in the top-level that instantiates the clients and network, -// probably using a TileLinkClientPort module. -// By eliding the header subbundles within the clients we can enable -// hierarchical P&R while minimizing unconnected port errors in GDS. -// Secondly, this version of the interface elides Finish messages, with the -// assumption that a FinishUnit has been coupled to the TileLinkIO port -// to deal with acking received Grants. +/** This version of UncachedTileLinkIO does not contain network headers. + * It is intended for use within client agents. + * + * Headers are provided in the top-level that instantiates the clients and network, + * probably using a [[uncore.ClientTileLinkPort]] module. + * By eliding the header subbundles within the clients we can enable + * hierarchical P-and-R while minimizing unconnected port errors in GDS. + * + * Secondly, this version of the interface elides [[uncore.Finish]] messages, with the + * assumption that a [[uncore.FinishUnit]] has been coupled to the TileLinkIO port + * to deal with acking received [[uncore.Grants]]. + */ class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } +/** This version of TileLinkIO does not contain network headers. + * It is intended for use within client agents. + */ class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } +/** This version of TileLinkIO does not contain network headers, but + * every channel does include an extra client_id subbundle. + * It is intended for use within Management agents. + * + * Managers need to track where [[uncore.Acquire]] and [[uncore.Release]] messages + * originated so that they can send a [[uncore.Grant]] to the right place. + * Similarly they must be able to issues Probes to particular clients. + * However, we'd still prefer to have [[uncore.ManagerTileLinkPort]] fill in + * the header.src to enable hierarchical p-and-r of the managers. Additionally, + * coherent clients might be mapped to random network port ids, and we'll leave it to the + * [[uncore.ManagerTileLinkPort]] to apply the correct mapping. Managers do need to + * see Finished so they know when to allow new transactions on a cache + * block to proceed. + */ class ManagerTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new AcquireFromSrc).flip val grant = new DecoupledIO(new GrantToDst) @@ -544,17 +736,7 @@ class ManagerTileLinkIO extends TLBundle { val release = new DecoupledIO(new ReleaseFromSrc).flip } -class ClientTileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ object TileLinkIOWrapper { def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { val conv = Module(new ClientTileLinkIOWrapper)(p) @@ -580,13 +762,32 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -class FinishQueueEntry extends TLBundle { - val fin = new Finish - val dst = UInt(width = log2Up(params(LNEndpoints))) +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) } -class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +class ClientTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip @@ -633,36 +834,23 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with Has } } -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) } -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probes]]. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { val client = new ClientTileLinkIO().flip @@ -686,6 +874,31 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extend io.client.grant <> gnt_without_header } +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { val io = new Bundle { val manager = new ManagerTileLinkIO().flip @@ -700,8 +913,25 @@ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extend io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** Struct for describing per-channel queue depths */ case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip @@ -905,10 +1135,10 @@ class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiter } /** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular TileLinkChannel or pair of channels. + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. * * Won't count message types that don't have data. - * Used in XactTrackers and FinishUnit. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. */ trait HasDataBeatCounters { type HasBeat = TileLinkChannel with HasTileLinkBeatId @@ -926,7 +1156,7 @@ trait HasDataBeatCounters { (cnt, done) } - /** Counter for beats on outgoing DecoupledIOs */ + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = connectDataBeatCounter(in.fire(), in.bits, beat) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8dda15b7..d7573aec 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,15 +2,10 @@ package uncore import Chisel._ -import scala.reflect._ -import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NIncoherentClients extends Field[Int] -case object NCoherentClients extends Field[Int] -case object L2CoherencePolicy extends Field[CoherencePolicy] trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 @@ -28,6 +23,7 @@ trait CoherenceAgentParameters extends UsesParameters { val innerByteAddrBits = log2Up(innerDataBits/8) require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } + abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters @@ -53,7 +49,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) - val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCachingClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits def irel(dummy: Int = 0) = inner.release.bits @@ -129,5 +125,5 @@ abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = - ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) + ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From f66a9fd7a6fd8f438c823cae85b24fd7cc4cd0c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 10:46:02 -0700 Subject: [PATCH 369/374] simplify ClientMetadata.makeRelease --- uncore/src/main/scala/metadata.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index cf52d286..f91035f4 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -59,13 +59,12 @@ class ClientMetadata extends CoherenceMetadata { def makeRelease( prb: Probe, - client_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Release = { Bundle(Release( voluntary = Bool(false), r_type = co.getReleaseType(prb, this), - client_xact_id = client_xact_id, + client_xact_id = UInt(0), addr_block = prb.addr_block, addr_beat = addr_beat, data = data), { case TLId => id }) From 4c7969b2b37bafa83dac60b1a20c6288110d958b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 16:32:09 -0700 Subject: [PATCH 370/374] Metadata docs and api cleanup --- uncore/src/main/scala/cache.scala | 25 ++-- uncore/src/main/scala/metadata.scala | 173 ++++++++++++++++++++++----- uncore/src/main/scala/tilelink.scala | 7 ++ 3 files changed, 160 insertions(+), 45 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75bb9e1f..4adb09a3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -338,10 +338,6 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2SecondaryMissInfo extends TLBundle - with HasTileLinkBeatId - with HasClientTransactionId - class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -617,10 +613,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_old_meta.coh.clone } // Secondary miss queue - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - // TODO add ignt.dst <- iacq.src + val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) @@ -741,6 +734,12 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { can_merge_iacq_put || can_merge_iacq_get + // Enqueue secondary miss information + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + // TODO add ignt.dst <- iacq.src + // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) @@ -792,22 +791,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) - ignt_q.io.enq.valid := iacq_data_done ignt_q.io.deq.ready := ignt_data_done - // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) + // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( - dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src - acq = xact, - client_xact_id = ignt_q.io.deq.bits.client_xact_id, + pri = xact, + sec = ignt_q.io.deq.bits, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, data_buffer(ignt_data_idx))) + io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( pending_coh.inner.onGrant(io.ignt()), diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index f91035f4..74e410c6 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers +/** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata extends Bundle { val co = params(TLCoherencePolicy) val id = params(TLId) @@ -15,21 +15,35 @@ abstract class CoherenceMetadata extends Bundle { * memory operations or [[uncore.Probe]] messages. */ class ClientMetadata extends CoherenceMetadata { + /** Actual state information stored in this bundle */ val state = UInt(width = co.clientStateWidth) + /** Metadata equality */ def ===(rhs: ClientMetadata): Bool = this.state === rhs.state def !=(rhs: ClientMetadata): Bool = !this.===(rhs) + /** Is the block's data present in this cache */ def isValid(dummy: Int = 0): Bool = co.isValid(this) - def isHit(cmd: UInt): Bool = co.isHit(cmd, this) - def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) - def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = - co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) - def requiresReleaseOnCacheControl(cmd: UInt): Bool = - co.requiresReleaseOnCacheControl(cmd: UInt, this) + /** Does this cache have permissions on this block sufficient to perform op */ + def isHit(op_code: UInt): Bool = co.isHit(op_code, this) + /** Does this cache lack permissions on this block sufficient to perform op */ + def isMiss(op_code: UInt): Bool = !co.isHit(op_code, this) + /** Does a secondary miss on the block require another Acquire message */ + def requiresAcquireOnSecondaryMiss(first_op: UInt, second_op: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_op, second_op, this) + /** Does op require a Release to be made to outer memory */ + def requiresReleaseOnCacheControl(op_code: UInt): Bool = + co.requiresReleaseOnCacheControl(op_code: UInt, this) + /** Does an eviction require a Release to be made to outer memory */ def requiresVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresReleaseOnCacheControl(M_FLUSH, this) + /** Constructs an Acquire message based on this metdata and a memory operation + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ def makeAcquire( client_xact_id: UInt, addr_block: UInt, @@ -43,6 +57,13 @@ class ClientMetadata extends CoherenceMetadata { { case TLId => id }) } + /** Constructs a Release message based on this metadata on an eviction + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being written back + */ def makeVoluntaryWriteback( client_xact_id: UInt, addr_block: UInt, @@ -57,6 +78,12 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Constructs a Release message based on this metadata and a [[uncore.Probe]] + * + * @param the incoming [[uncore.Probe]] + * @param addr_beat sub-block address (which beat) + * @param data data being released + */ def makeRelease( prb: Probe, addr_beat: UInt = UInt(0), @@ -70,16 +97,37 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** New metadata after receiving a [[uncore.Grant]] + * + * @param incoming the incoming [[uncore.Grant]] + * @param pending the mem op that triggered this transaction + */ def onGrant(incoming: Grant, pending: UInt): ClientMetadata = Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param incoming the incoming [[uncore.Probe]] + */ def onProbe(incoming: Probe): ClientMetadata = Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) - def onHit(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) - def onCacheControl(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) + + /** New metadata after a op_code hits this block + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onHit(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onCacheControl(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(op_code, this), { case TLId => id }) } +/** Factories for ClientMetadata, including on reset */ object ClientMetadata { def apply(state: UInt) = { val meta = new ClientMetadata @@ -89,33 +137,64 @@ object ClientMetadata { def onReset = new ClientMetadata().co.clientMetadataOnReset } -/* The ManagerMetadata stores manager-side information about the status - of a cache block, including whether it has any known sharers. Its - API can be used to create Probe and Grant TileLink messages. -*/ +/** Stores manager-side information about the status + * of a cache block, including whether it has any known sharers. + * + * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. + */ class ManagerMetadata extends CoherenceMetadata { + // Currently no coherence policies assume manager-side state information // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + + /** The directory information for this block */ val sharers = UInt(width = co.dir.width) + /** Metadata equality */ def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) - def full(dummy: Int = 0) = co.dir.full(this.sharers) + /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ + def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) + + /** Does this [[uncore.Acquire]] require [[uncore.Probes]] to be sent */ def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) - def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + /** Does this memory op require [[uncore.Probes]] to be sent */ + def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) + /** Does an eviction require [[uncore.Probes]] to be sent */ def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - + /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] + * + * @param dst Destination client id for this Probe + * @param acq Acquire message triggering this Probe + */ def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op + * + * @param dst Destination client id for this Probe + * @param op_code memory operation triggering this Probe + * @param addr_block address of the cache block being probed + */ + def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block), { case TLId => id }) + + /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction + * + * @param dst Destination client id for this Probe + * @param addr_block address of the cache block being probed prior to eviction + */ def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = makeProbe(dst, M_FLUSH, addr_block) + /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] + * + * @param rel Release message being acknowledged by this Grant + * @param manager_xact_id manager's transaction id + */ def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( dst = rel.client_id, @@ -125,6 +204,15 @@ class ManagerMetadata extends CoherenceMetadata { manager_xact_id = manager_xact_id), { case TLId => id }) } + /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] + * + * May contain single or multiple beats of data, or just be a permissions upgrade. + * + * @param acq Acquire message being responded to by this Grant + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ def makeGrant( acq: AcquireFromSrc, manager_xact_id: UInt, @@ -142,26 +230,42 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides + * + * Used to respond to secondary misses merged into this transaction. + * May contain single or multiple beats of data. + * + * @param pri Primary miss's Acquire message, used to get g_type and dst + * @param sec Secondary miss info, used to get beat and client_xact_id + * @param manager_xact_id manager's transaction id + * @param data data being refilled to the original requestor + */ def makeGrant( - dst: UInt, - acq: AcquireFromSrc, - client_xact_id: UInt, + pri: AcquireFromSrc, + sec: SecondaryMissInfo, manager_xact_id: UInt, - addr_beat: UInt, data: UInt): GrantToDst = { - val g = makeGrant(acq, manager_xact_id, addr_beat, data) - g.client_id := dst - g.client_xact_id := client_xact_id + val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) + g.client_xact_id := sec.client_xact_id g } + /** New metadata after receiving a [[uncore.ReleaseFromSrc]] + * + * @param incoming the incoming [[uncore.ReleaseFromSrc]] + */ def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) + /** New metadata after sending a [[uncore.GrantToDst]] + * + * @param outgoing the outgoing [[uncore.GrantToDst]] + */ def onGrant(outgoing: GrantToDst): ManagerMetadata = Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } +/** Factories for ManagerMetadata, including on reset */ object ManagerMetadata { def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { val meta = new ManagerMetadata @@ -178,10 +282,14 @@ object ManagerMetadata { def onReset = new ManagerMetadata().co.managerMetadataOnReset } -/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy - that is a Manager with respect to some inner caches and a Client with - respect to some outer cache. -*/ +/** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + * that is a manager with respect to some inner caches and a client with + * respect to some outer cache. + * + * This class makes use of two different sets of TileLink parameters, which are + * applied by contextually mapping [[uncore.TLId]] to one of + * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. + */ class HierarchicalMetadata extends CoherenceMetadata { val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) @@ -190,6 +298,7 @@ class HierarchicalMetadata extends CoherenceMetadata { def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } +/** Factories for HierarchicalMetadata, including on reset */ object HierarchicalMetadata { def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { val m = new HierarchicalMetadata @@ -200,5 +309,7 @@ object HierarchicalMetadata { def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) } +/** Identifies the TLId of the inner network in a hierarchical cache controller */ case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b759d48b..1c1c2bb4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -785,6 +785,13 @@ class ClientTileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } +/** Used to track metadata for transactions where multiple secondary misses have been merged + * and handled by a single transaction tracker. + */ +class SecondaryMissInfo extends TLBundle // TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId + with HasTileLinkBeatId + with HasClientTransactionId + /** A helper module that automatically issues [[uncore.Finish]] messages in repsonse * to [[uncore.Grant]] that it receives from a manager and forwards to a client */ From 11b5222d014a425ac65545491e92800b69c01336 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Apr 2015 22:23:04 -0700 Subject: [PATCH 371/374] Refactored WritebackUnit --- uncore/src/main/scala/cache.scala | 245 +++++++++++++----------------- 1 file changed, 105 insertions(+), 140 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4adb09a3..32529202 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -192,9 +192,8 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && in.bits.id === UInt(i) } } } @@ -621,7 +620,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 - val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectTwoWayBeatCounter( max = 1, up = io.outer.acquire, @@ -646,7 +645,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { pending_writes.orR || pending_resps.orR || pending_puts.orR || - pending_ognts || + pending_ognt || ignt_q.io.count > UInt(0) || //pending_meta_write || // Has own state: s_meta_write pending_ifins) @@ -794,8 +793,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) + (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( pri = xact, @@ -826,9 +824,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && - pending_reads.orR && - !pending_ognts + io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) @@ -847,7 +843,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val curr_write_beat = PriorityEncoder(pending_writes) io.data.write.valid := state === s_busy && pending_writes.orR && - !pending_ognts && + !pending_ognt && !pending_reads(curr_write_beat) && !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) @@ -870,10 +866,11 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie @@ -961,9 +958,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // These IOs are used for routing in the parent val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) + io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? @@ -974,10 +969,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { "AcquireTracker accepted data beat from different network source than initial request.") } -class L2WritebackReq extends L2HellaCacheBundle - with HasL2Id { - val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size - val coh = new HierarchicalMetadata +class L2WritebackReq extends L2Metadata with HasL2Id { + val idx = Bits(width = idxBits) val way_en = Bits(width = nWays) } @@ -996,136 +989,108 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) + val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) - val xact_coh = Reg{ new HierarchicalMetadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val xact_id = Reg{ UInt() } + val xact = Reg(new L2WritebackReq) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val xact_addr_block = Cat(xact.tag, xact.idx) - val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.wb.req.bits.coh.inner.full() - val mask_incoherent = full_sharers & ~io.incoherent.toBits + val pending_irels = + connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 + val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = + connectTwoWayBeatCounter(max = 1, up = io.outer.release, down = io.outer.grant) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp) + // Start the writeback sub-transaction + io.wb.req.ready := state === s_idle - val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) - val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write - val pending_ofin_on_ognt = io.ognt().makeFinish() + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := xact.coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := !io.irel().isVoluntary() && - io.irel().conflicts(xact_addr_block) && - (state === s_inner_probe) + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) + val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) + io.inner.release.ready := state === s_inner_probe || state === s_busy + when(io.inner.release.fire()) { + xact.coh.inner := inner_coh_on_irel + when(io.irel().hasData()) { xact.coh.outer := outer_coh_on_irel } // WB is a write + data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data + } - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) // default - io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.grant.ready := Bool(false) // default - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) + // If a release didn't write back data, have to read it from data array + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release)) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.way_en := xact.way_en + io.data.read.bits.addr_idx := xact.idx + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) - io.wb.req.ready := Bool(false) - io.wb.resp.valid := Bool(false) - io.wb.resp.bits.id := xact_id - - switch (state) { - is(s_idle) { - io.wb.req.ready := Bool(true) - when(io.wb.req.valid) { - xact_addr_block := io.wb.req.bits.addr_block - xact_coh := io.wb.req.bits.coh - xact_way_en := io.wb.req.bits.way_en - xact_id := io.wb.req.bits.id - irel_had_data := Bool(false) - val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_inner_probes) { - pending_probes := mask_incoherent - irel_cnt := PopCount(mask_incoherent) - } - state := Mux(needs_inner_probes, s_inner_probe, s_data_read) - } - } - is(s_inner_probe) { - // Send probes - io.inner.probe.valid := pending_probes != UInt(0) - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - irel_had_data := Bool(true) - xact_coh.outer := pending_ocoh_on_irel - data_buffer(io.irel().addr_beat) := io.irel().data - } - // We don't decrement irel_cnt until we've received all the data beats. - when(!io.irel().hasData() || irel_data_done) { - irel_cnt := irel_cnt - UInt(1) - } - } - when(irel_cnt === UInt(0)) { - state := Mux(irel_had_data, // If someone released a dirty block - s_outer_release, // write that block back, otherwise - Mux(xact_coh.outer.requiresVoluntaryWriteback(), - s_data_read, // write extant dirty data back, or just - s_wb_resp)) // drop a clean block after collecting acks - } - } - is(s_data_read) { - io.data.read.valid := Bool(true) - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(read_data_done) { state := s_data_resp } - } - is(s_data_resp) { - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_release } - } - is(s_outer_release) { - io.outer.release.valid := Bool(true) - when(orel_data_done) { - state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) - } - } - is(s_outer_grant) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - state := s_wb_resp - } - } - is(s_wb_resp) { - io.wb.resp.valid := Bool(true) - state := s_idle - } + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + when(io.data.resp.valid) { + data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + + // Once the data is buffered we can write it back to outer memory + pending_orel_data := pending_orel_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitInternal(io.data.resp) + io.outer.release.valid := state === s_busy && + (!io.orel().hasData() || pending_orel_data(orel_data_idx)) + io.outer.release.bits := xact.coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_idx, + data = data_buffer(orel_data_idx)) + + // Wait for an acknowledgement + io.outer.grant.ready := state === s_outer_grant + + // Respond to the initiating transaction handler signalling completion of the writeback + io.wb.resp.valid := state === s_wb_resp + io.wb.resp.bits.id := xact.id + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.wb.req.valid) { + xact := io.wb.req.bits + val coh = io.wb.req.bits.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } + pending_reads := SInt(-1, width = innerDataBeats) + pending_resps := UInt(0) + pending_orel_data := UInt(0) + state := Mux(needs_inner_probes, s_inner_probe, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) + } + when(state === s_busy && orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) + } + when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } + when(state === s_wb_resp ) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // IOs pinned low + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) // TODO: handle these here + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.finish.ready := Bool(false) } From 09e30041edd97d9e3e6dfa0bb779c64c4234a770 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 12:56:33 -0700 Subject: [PATCH 372/374] Voluntary Writeback tracker rewrite --- uncore/src/main/scala/cache.scala | 212 ++++++++++++++---------------- 1 file changed, 97 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 32529202..7fcd1408 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -337,8 +337,7 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCacheBank extends HierarchicalCoherenceAgent - with L2HellaCacheParameters { +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) @@ -357,8 +356,7 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile extends L2HellaCacheModule - with HasCoherenceAgentWiringHelpers { +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions @@ -381,7 +379,6 @@ class TSHRFile extends L2HellaCacheModule val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) - val block_acquires = acquireConflicts.orR io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { @@ -432,7 +429,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { - class CacheBlockBuffer { + class CacheBlockBuffer { // TODO val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) @@ -477,127 +474,118 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { val isPartial = a.wmask() != Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[T] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[T] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } } class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val xact_tag_match = Reg{ Bool() } - val xact_old_meta = Reg{ new L2Metadata } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh - val collect_irel_data = Reg(init=Bool(false)) - val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := io.irel().isVoluntary() + val all_pending_done = + !(pending_writes.orR || + pending_ignt) - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := state === s_idle || pending_irels.orR + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } - io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) - io.meta.read.valid := Bool(false) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + + // Write the voluntarily written back data to this cache + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && pending_writes.orR + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(curr_write_beat) + + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + + // End a transaction by updating the block metadata + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) - io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write - io.wb.req.valid := Bool(false) + io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), + xact_old_meta.coh.outer.onHit(M_XWR), + xact_old_meta.coh.outer) - when(collect_irel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid(io.irel().addr_beat) := Bool(true) + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.valid) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) } - when(irel_data_done) { collect_irel_data := Bool(false) } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_meta_read } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := s_busy + } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := io.irel() - data_buffer(io.irel().addr_beat) := io.irel().data - collect_irel_data := io.irel().hasMultibeatData() - irel_data_valid := io.irel().hasData() << io.irel().addr_beat - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_old_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_inner_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_inner_grant) { - io.inner.grant.valid := !collect_irel_data - when(io.inner.grant.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } + // These IOs are used for routing in the parent + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) // Checks for illegal behavior assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) @@ -742,6 +730,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + + // Issue a request to the writeback unit + io.wb.req.valid := state === s_wb_req + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag + io.wb.req.bits.coh := xact_old_meta.coh + io.wb.req.bits.way_en := xact_way_en + // Track which clients yet need to be probed and make Probe message pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) @@ -773,10 +775,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { op_code = xact.op_code()), Bundle(Acquire(xact))(outerTLParams)) - // Probes from the outer memory system are handled in the WritebackUnit - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - + // Handle the response from outer memory io.outer.grant.ready := state === s_busy val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, @@ -853,11 +852,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := state === s_meta_read - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - + // End a transaction by updating the block metadata io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -865,13 +860,6 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := state === s_wb_req - io.wb.req.bits.id := UInt(trackerId) - io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.wb.req.bits.tag := xact_old_meta.tag - io.wb.req.bits.coh := xact_old_meta.coh - io.wb.req.bits.way_en := xact_way_en - // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie val beat = io.iacq().addr_beat @@ -988,6 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO + pinAllReadyValidLow(io) val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) @@ -1086,11 +1075,4 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) - - // IOs pinned low - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) // TODO: handle these here - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.finish.ready := Bool(false) } From 3673295d03672b2a1fcc96e7b402de571c7bf7ee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 16:59:30 -0700 Subject: [PATCH 373/374] network shim cleanup --- uncore/src/main/scala/network.scala | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index f61cec6c..4b00091d 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -57,7 +57,10 @@ class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +72,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload @@ -77,3 +80,25 @@ object DecoupledLogicalNetworkIOUnwrapper { out } } + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Decoupled(new LogicalNetworkIO(in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Decoupled(new PhysicalNetworkIO(n, in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} From 4cef8c9cd4b1334d9705932f3ba226960b0dabd3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 10:55:38 -0700 Subject: [PATCH 374/374] Added MemIOArbiter --- uncore/src/main/scala/memserdes.scala | 65 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7497e2e4..a4944e1e 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -60,14 +60,14 @@ class MemPipeIO extends Bundle { val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int) extends Bundle -{ +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends MIFModule +class MemSerdes extends MIFModule { + val w = params(HTIFWidth) val io = new Bundle { val wide = new MemIO().flip val narrow = new MemSerializedIO(w) @@ -517,6 +517,65 @@ object HellaQueue } } +class MemIOArbiter(val arbN: Int) extends MIFModule { + val io = new Bundle { + val inner = Vec.fill(arbN){new MemIO}.flip + val outer = new MemIO + } + + if(arbN > 1) { + val cmd_arb = Module(new RRArbiter(new MemReqCmd, arbN)) + val choice_q = Module(new Queue(cmd_arb.io.chosen.clone, 4)) + val (data_cnt, data_done) = Counter(io.outer.req_data.fire(), mifDataBeats) + + io.inner.map(_.req_cmd).zipWithIndex.zip(cmd_arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.tag := Cat(req.bits.tag, UInt(id)) + req.ready := arb.ready + }} + io.outer.req_cmd.bits := cmd_arb.io.out.bits + io.outer.req_cmd.valid := cmd_arb.io.out.valid && choice_q.io.enq.ready + cmd_arb.io.out.ready := io.outer.req_cmd.ready && choice_q.io.enq.ready + choice_q.io.enq.bits := cmd_arb.io.chosen + choice_q.io.enq.valid := cmd_arb.io.out.fire() && cmd_arb.io.out.bits.rw + + io.outer.req_data.bits := io.inner(choice_q.io.deq.bits).req_data.bits + io.outer.req_data.valid := io.inner(choice_q.io.deq.bits).req_data.valid && choice_q.io.deq.valid + io.inner.map(_.req_data.ready).zipWithIndex.foreach { + case(r, i) => r := UInt(i) === choice_q.io.deq.bits && choice_q.io.deq.valid + } + choice_q.io.deq.ready := data_done + + io.outer.resp.ready := Bool(false) + for (i <- 0 until arbN) { + io.inner(i).resp.valid := Bool(false) + when(io.outer.resp.bits.tag(log2Up(arbN)-1,0).toUInt === UInt(i)) { + io.inner(i).resp.valid := io.outer.resp.valid + io.outer.resp.ready := io.inner(i).resp.ready + } + io.inner(i).resp.bits := io.outer.resp.bits + io.inner(i).resp.bits.tag := io.outer.resp.bits.tag >> UInt(log2Up(arbN)) + } + } else { io.inner.head <> io.outer } +} + +object MemIOMemPipeIOConverter { + def apply(in: MemPipeIO): MemIO = { + val out = new MemIO().asDirectionless + in.resp.valid := out.resp.valid + in.resp.bits := out.resp.bits + out.resp.ready := Bool(true) + out.req_cmd.valid := in.req_cmd.valid + out.req_cmd.bits := in.req_cmd.bits + in.req_cmd.ready := out.req_cmd.ready + out.req_data.valid := in.req_data.valid + out.req_data.bits := in.req_data.bits + in.req_data.ready := out.req_data.ready + out + } +} + class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip